Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/logs/upgrade-proxysql-5-7.log
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
+ CLUSTER=upgrade-proxysql
+ CLUSTER_SIZE=3
+ TARGET_OPERATOR_VER=1.20.0
+ TARGET_IMAGE=perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd
+ TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc5.7
+ TARGET_IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest
+ TARGET_IMAGE_PROXY=perconalab/percona-xtradb-cluster-operator:main-proxysql
+ TARGET_IMAGE_HAPROXY=perconalab/percona-xtradb-cluster-operator:main-haproxy
+ TARGET_IMAGE_BACKUP=perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup
+ TARGET_IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]]
++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/'
++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc5.7
+ PXC_VER=5.7
++ tail -n1
++ sort -V
++ jq -r '.versions[].operator'
++ curl -s https://check.percona.com/versions/v1/pxc-operator
+ INIT_OPERATOR_VER=1.19.0
+ [[ 1.19.0 == \1\.\2\0\.\0 ]]
+ GIT_TAG=v1.19.0
++ curl -s 'https://check.percona.com/versions/v1/pxc-operator/1.19.0/latest?databaseVersion=5.7'
+ INIT_OPERATOR_IMAGES='{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
+ OPERATOR_NAME=percona-xtradb-cluster-operator
++ jq -r '.versions[].matrix.operator[].imagePath'
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
+ IMAGE=percona/percona-xtradb-cluster-operator:1.19.0
++ echo perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd
++ cut -d/ -f1
+ [[ perconalab == \p\e\r\c\o\n\a\l\a\b ]]
+ IMAGE=perconalab/percona-xtradb-cluster-operator:1.19.0
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
++ jq -r '.versions[].matrix.pxc[].imagePath'
+ IMAGE_PXC=percona/percona-xtradb-cluster:5.7.44-31.65
++ jq -r '.versions[].matrix.pmm[].imagePath'
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
++ tail -n1
+ IMAGE_PMM_CLIENT=percona/pmm-client:3.5.0
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
++ jq -r '.versions[].matrix.proxysql[].imagePath'
+ IMAGE_PROXY=percona/proxysql2:2.7.3-1.2
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
++ jq -r '.versions[].matrix.haproxy[].imagePath'
+ IMAGE_HAPROXY=percona/haproxy:2.8.17
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
++ jq -r '.versions[].matrix.backup[].imagePath'
+ IMAGE_BACKUP=percona/percona-xtrabackup:2.4.29
++ jq -r '.versions[].matrix.logCollector[].imagePath'
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
+ IMAGE_LOGCOLLECTOR=percona/fluentbit:4.0.1-1
+ [[ -n '' ]]
+ [[ 1.20.0 == \1\.\1\9\.\0 ]]
+ main
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.1Who5uCrRQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.VnAn8KkeGU
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.1Who5uCrRQ
namespace/cert-manager created
+ cat /tmp/tmp.VnAn8KkeGU
+ rm /tmp/tmp.1Who5uCrRQ /tmp/tmp.VnAn8KkeGU
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.J43XYxjIGm
++ mktemp
+ local LAST_ERR=/tmp/tmp.Is3hcJmuvL
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.J43XYxjIGm
namespace/cert-manager labeled
+ cat /tmp/tmp.Is3hcJmuvL
+ rm /tmp/tmp.J43XYxjIGm /tmp/tmp.Is3hcJmuvL
+ return 0
+ kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.I2RxcfwRdm
++ mktemp
+ local LAST_ERR=/tmp/tmp.T818cuWpnl
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.I2RxcfwRdm
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-tokenrequest created
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager-cainjector created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.T818cuWpnl
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.I2RxcfwRdm /tmp/tmp.T818cuWpnl
+ return 0
+ '[' '' == 4.10 ']'
+ sleep 70
+ create_infra_gh upgrade-proxysql-4143 v1.19.0
+ local ns=upgrade-proxysql-4143
+ local git_tag=v1.19.0
+ '[' -n pxc-operator ']'
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ awk '-F ' '{print $2}'
++ tail -n1
++ sed s/NAMESPACE//
++ helm list --all-namespaces --filter chaos-mesh
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrolebinding
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrole
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ xargs kubectl delete ns
+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ kubectl_bin get ns
++ mktemp
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.KZtJd3wIlH
++ mktemp
+ local LAST_OUT=/tmp/tmp.yFoVpJW8z9
++ mktemp
+ local LAST_ERR=/tmp/tmp.YQxAWEYXIS
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
++ mktemp
+ local LAST_ERR=/tmp/tmp.WEL3HDzdRc
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.KZtJd3wIlH
+ cat /tmp/tmp.WEL3HDzdRc
+ rm /tmp/tmp.KZtJd3wIlH /tmp/tmp.WEL3HDzdRc
+ return 0
namespace "cert-manager" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.yFoVpJW8z9
namespace "pxc-operator" deleted
+ cat /tmp/tmp.YQxAWEYXIS
+ rm /tmp/tmp.yFoVpJW8z9 /tmp/tmp.YQxAWEYXIS
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.T8a4EKbYc1
++ mktemp
+ local LAST_ERR=/tmp/tmp.aLitiKlpeR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.T8a4EKbYc1
namespace/pxc-operator created
+ cat /tmp/tmp.aLitiKlpeR
+ rm /tmp/tmp.T8a4EKbYc1 /tmp/tmp.aLitiKlpeR
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aZ3uIOVd1b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZAxvAYZADP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.aZ3uIOVd1b
++ cat /tmp/tmp.ZAxvAYZADP
++ rm /tmp/tmp.aZ3uIOVd1b /tmp/tmp.ZAxvAYZADP
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster5 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.Oman9Manq7
++ mktemp
+ local LAST_ERR=/tmp/tmp.HtZk1Y8xbU
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster5 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Oman9Manq7
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster5" modified.
+ cat /tmp/tmp.HtZk1Y8xbU
+ rm /tmp/tmp.Oman9Manq7 /tmp/tmp.HtZk1Y8xbU
+ return 0
+ deploy_operator_gh v1.19.0
+ local git_tag=v1.19.0
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

++ kubectl_bin get crds -o 'jsonpath={.items[?(@.metadata.name == "perconaxtradbclusters.pxc.percona.com")].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Vyz7dyxYeV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SmCa8qPG2c
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crds -o 'jsonpath={.items[?(@.metadata.name == "perconaxtradbclusters.pxc.percona.com")].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Vyz7dyxYeV
++ cat /tmp/tmp.SmCa8qPG2c
++ rm /tmp/tmp.Vyz7dyxYeV /tmp/tmp.SmCa8qPG2c
++ return 0
+ [[ -n perconaxtradbclusters.pxc.percona.com ]]
++ kubectl_bin get crd/perconaxtradbclusters.pxc.percona.com -o 'jsonpath={.spec.versions[?(@.name == "v1-19-0")].name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hOyWYYqLX8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.e4BgblUUOP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/perconaxtradbclusters.pxc.percona.com -o 'jsonpath={.spec.versions[?(@.name == "v1-19-0")].name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hOyWYYqLX8
++ cat /tmp/tmp.e4BgblUUOP
++ rm /tmp/tmp.hOyWYYqLX8 /tmp/tmp.e4BgblUUOP
++ return 0
+ [[ -n '' ]]
+ kubectl_bin apply --server-side --force-conflicts -f https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.19.0/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.SJgYa0suvi
++ mktemp
+ local LAST_ERR=/tmp/tmp.U7vUX49yIY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.19.0/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.SJgYa0suvi
+ cat /tmp/tmp.U7vUX49yIY
+ rm /tmp/tmp.SJgYa0suvi /tmp/tmp.U7vUX49yIY
+ return 0
+ local rbac_yaml=rbac
+ local operator_yaml=operator.yaml
+ '[' -n pxc-operator ']'
+ rbac_yaml=cw-rbac
+ operator_yaml=cw-operator.yaml
+ apply_rbac_gh cw-rbac v1.19.0
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ local git_tag=v1.19.0
+ curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.19.0/deploy/cw-rbac.yaml
+ /usr/bin/sed -i -e 's^namespace: .*^namespace: pxc-operator^' /tmp/tmp.9XEC036CwX/rbac_v1.19.0.yaml
+ kubectl_bin apply -f /tmp/tmp.9XEC036CwX/rbac_v1.19.0.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ob5by5J8GS
++ mktemp
+ local LAST_ERR=/tmp/tmp.nk2xHOkb9G
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /tmp/tmp.9XEC036CwX/rbac_v1.19.0.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ob5by5J8GS
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.nk2xHOkb9G
+ rm /tmp/tmp.ob5by5J8GS /tmp/tmp.nk2xHOkb9G
+ return 0
+ curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.19.0/deploy/cw-operator.yaml
+ cat /tmp/tmp.9XEC036CwX/cw-operator.yaml_v1.19.0.yaml
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:1.19.0^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"'
+ kubectl_bin apply -n pxc-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.bte0lYNUlA
++ mktemp
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"'
+ local LAST_ERR=/tmp/tmp.bLr3o9fOAG
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n pxc-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.bte0lYNUlA
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.bLr3o9fOAG
+ rm /tmp/tmp.bte0lYNUlA /tmp/tmp.bLr3o9fOAG
+ return 0
+ sleep 10
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ head -1
+++ mktemp
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ local LAST_OUT=/tmp/tmp.jfvdKYRWCV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.f374TWmVt5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.jfvdKYRWCV
++ cat /tmp/tmp.f374TWmVt5
++ rm /tmp/tmp.jfvdKYRWCV /tmp/tmp.f374TWmVt5
++ return 0
+ wait_pod percona-xtradb-cluster-operator-7f865c95d-ct4q4
+ local pod=percona-xtradb-cluster-operator-7f865c95d-ct4q4
+ local max_retry=480
+ local ns=
++ echo percona-xtradb-cluster-operator-7f865c95d-ct4q4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-7f865c95d-ct4q4 condition met
waiting for pod/percona-xtradb-cluster-operator-7f865c95d-ct4q4 to become Ready.Ok
+ create_namespace upgrade-proxysql-4143
+ local namespace=upgrade-proxysql-4143
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces upgrade-proxysql-4143'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces upgrade-proxysql-4143
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace upgrade-proxysql-4143
++ mktemp
+ awk '{print$1}'
+ kubectl_bin get ns
+ local LAST_OUT=/tmp/tmp.EhoJLWZu0V
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.45jVLszfcZ
+ local exit_status=0
+ local LAST_OUT=/tmp/tmp.LIyVEaY4uC
++ seq 0 2
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-proxysql-4143
+ local LAST_ERR=/tmp/tmp.1QqClTmf77
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-proxysql-4143
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.LIyVEaY4uC
+ cat /tmp/tmp.1QqClTmf77
+ rm /tmp/tmp.LIyVEaY4uC /tmp/tmp.1QqClTmf77
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-proxysql-4143
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.EhoJLWZu0V
+ cat /tmp/tmp.45jVLszfcZ
Error from server (NotFound): namespaces "upgrade-proxysql-4143" not found
+ rm /tmp/tmp.EhoJLWZu0V /tmp/tmp.45jVLszfcZ
+ return 1
+ :
+ wait_for_delete namespace/upgrade-proxysql-4143
+ local res=namespace/upgrade-proxysql-4143
+ echo -n 'waiting for namespace/upgrade-proxysql-4143 to be deleted'
waiting for namespace/upgrade-proxysql-4143 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "upgrade-proxysql-4143" not found
+ desc 'create namespace upgrade-proxysql-4143'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace upgrade-proxysql-4143
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace upgrade-proxysql-4143
++ mktemp
+ local LAST_OUT=/tmp/tmp.pYZhBSMrZ3
++ mktemp
+ local LAST_ERR=/tmp/tmp.h6r4dPMEjI
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace upgrade-proxysql-4143
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.pYZhBSMrZ3
namespace/upgrade-proxysql-4143 created
+ cat /tmp/tmp.h6r4dPMEjI
+ rm /tmp/tmp.pYZhBSMrZ3 /tmp/tmp.h6r4dPMEjI
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iSaufHFGgs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rJfLxJk23h
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.iSaufHFGgs
++ cat /tmp/tmp.rJfLxJk23h
++ rm /tmp/tmp.iSaufHFGgs /tmp/tmp.rJfLxJk23h
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster5 --namespace=upgrade-proxysql-4143
++ mktemp
+ local LAST_OUT=/tmp/tmp.phD5OtRLts
++ mktemp
+ local LAST_ERR=/tmp/tmp.epqaQjLnL0
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster5 --namespace=upgrade-proxysql-4143
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.phD5OtRLts
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster5" modified.
+ cat /tmp/tmp.epqaQjLnL0
+ rm /tmp/tmp.phD5OtRLts /tmp/tmp.epqaQjLnL0
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.FOKmqTPZbZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.xBEjLrgozp
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.FOKmqTPZbZ
secret/minio-secret created
secret/aws-s3-secret created
secret/do-spaces-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.xBEjLrgozp
+ rm /tmp/tmp.FOKmqTPZbZ /tmp/tmp.xBEjLrgozp
+ return 0
+ local proxy=proxysql
+ local cr_yaml=/tmp/tmp.9XEC036CwX/cr_1.19.0_proxysql.yaml
+ prepare_cr_yaml /tmp/tmp.9XEC036CwX/cr_1.19.0_proxysql.yaml proxysql upgrade-proxysql 3 v1.19.0
+ local cr_yaml=/tmp/tmp.9XEC036CwX/cr_1.19.0_proxysql.yaml
+ local proxy=proxysql
+ local cluster=upgrade-proxysql
+ local cluster_size=3
+ local git_tag=v1.19.0
+ curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.19.0/deploy/cr.yaml
+ yq eval '
			.metadata.name = "upgrade-proxysql" |
			.spec.secretsName = "my-cluster-secrets" |
			.spec.vaultSecretName = "some-name-vault" |
			.spec.sslSecretName = "some-name-ssl" |
			.spec.sslInternalSecretName = "some-name-ssl-internal" |
			.spec.upgradeOptions.apply = "disabled" |
			.spec.pxc.size = 3 |
			.spec.proxysql.size = 3 |
			.spec.haproxy.size = 3 |
			.spec.pxc.image = "-pxc" |
			.spec.proxysql.image = "-proxysql" |
			.spec.haproxy.image = "-haproxy" |
			.spec.backup.image = "-backup" |
			.spec.backup.storages.minio.s3.credentialsSecret = "minio-secret" |
			.spec.backup.storages.minio.s3.region = "us-east-1" |
			.spec.backup.storages.minio.s3.bucket = "operator-testing" |
			.spec.backup.storages.minio.s3.endpointUrl = "http://minio-service.#namespace:9000/" |
			.spec.backup.storages.minio.type = "s3" |
			.spec.pmm.image = "-pmm"
		' -
+ [[ proxysql == \h\a\p\r\o\x\y ]]
+ yq -i eval '
			.spec.haproxy.enabled = false |
			.spec.proxysql.enabled = true
		' /tmp/tmp.9XEC036CwX/cr_1.19.0_proxysql.yaml
+ [[ -n '' ]]
+ spinup_pxc upgrade-proxysql /tmp/tmp.9XEC036CwX/cr_1.19.0_proxysql.yaml 3 30 /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/secrets_without_tls.yml
+ local cluster=upgrade-proxysql
+ local config=/tmp/tmp.9XEC036CwX/cr_1.19.0_proxysql.yaml
+ local size=3
+ local sleep=30
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/secrets_without_tls.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/secrets_without_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.2UNcdGISYx
++ mktemp
+ local LAST_ERR=/tmp/tmp.N4EotW7vcB
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/secrets_without_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2UNcdGISYx
secret/my-cluster-secrets created
+ cat /tmp/tmp.N4EotW7vcB
+ rm /tmp/tmp.2UNcdGISYx /tmp/tmp.N4EotW7vcB
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/client.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/client.yml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/client.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/client.yml
+ local pvc_name=
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: percona/proxysql2:2.7.3-1.2#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.upgrade-proxysql-4143~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: percona/fluentbit:4.0.1-1#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:5.7.44-31.65#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:5.7.44-31.65#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:1.19.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: percona/pmm-client:3.5.0#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: percona/percona-xtrabackup:2.4.29#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: percona/haproxy:2.8.17#'
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ynRecYRu3m
++ mktemp
+ local LAST_ERR=/tmp/tmp.D4OyRpYFSs
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ynRecYRu3m
deployment.apps/pxc-client created
+ cat /tmp/tmp.D4OyRpYFSs
+ rm /tmp/tmp.ynRecYRu3m /tmp/tmp.D4OyRpYFSs
+ return 0
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ upgrade-proxysql == \d\e\m\a\n\d\-\b\a\c\k\u\p ]]
+ [[ upgrade-proxysql == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]]
+ apply_config /tmp/tmp.9XEC036CwX/cr_1.19.0_proxysql.yaml
+ local config_file=/tmp/tmp.9XEC036CwX/cr_1.19.0_proxysql.yaml
+ local pvc_name=
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /tmp/tmp.9XEC036CwX/cr_1.19.0_proxysql.yaml ''
+ local input_file=/tmp/tmp.9XEC036CwX/cr_1.19.0_proxysql.yaml
+ local pvc_name=
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.upgrade-proxysql-4143~
+ cat /tmp/tmp.9XEC036CwX/cr_1.19.0_proxysql.yaml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:1.19.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: percona/pmm-client:3.5.0#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: percona/haproxy:2.8.17#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: percona/fluentbit:4.0.1-1#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.nv7wvKgb8j
++ mktemp
+ /usr/bin/sed -e 's#image:.*-backup$#image: percona/percona-xtrabackup:2.4.29#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: percona/proxysql2:2.7.3-1.2#'
+ local LAST_ERR=/tmp/tmp.98OgsFi6k0
+ local exit_status=0
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:5.7.44-31.65#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:5.7.44-31.65#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nv7wvKgb8j
perconaxtradbcluster.pxc.percona.com/upgrade-proxysql created
+ cat /tmp/tmp.98OgsFi6k0
+ rm /tmp/tmp.nv7wvKgb8j /tmp/tmp.98OgsFi6k0
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy upgrade-proxysql
++ local target_cluster=upgrade-proxysql
+++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.2QWO6vfTaV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.j5q84tjamo
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.2QWO6vfTaV
+++ cat /tmp/tmp.j5q84tjamo
+++ rm /tmp/tmp.2QWO6vfTaV /tmp/tmp.j5q84tjamo
+++ return 0
++ [[ false == \t\r\u\e ]]
+++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0JtioZ8fpb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.gh5x5Ai5Sv
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.0JtioZ8fpb
+++ cat /tmp/tmp.gh5x5Ai5Sv
+++ rm /tmp/tmp.0JtioZ8fpb /tmp/tmp.gh5x5Ai5Sv
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo upgrade-proxysql-proxysql
++ return
+ local proxy=upgrade-proxysql-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n upgrade-proxysql-4143
++ mktemp
+ local LAST_OUT=/tmp/tmp.hrLOydgMPK
++ mktemp
+ local LAST_ERR=/tmp/tmp.icj6yE60kI
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n upgrade-proxysql-4143
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n upgrade-proxysql-4143
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n upgrade-proxysql-4143
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.hrLOydgMPK
+ cat /tmp/tmp.icj6yE60kI
error: no matching resources found
+ rm /tmp/tmp.hrLOydgMPK /tmp/tmp.icj6yE60kI
+ return 1
+ true
+ wait_for_running upgrade-proxysql-proxysql 1
+ local name=upgrade-proxysql-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-proxysql-0 480
+ local pod=upgrade-proxysql-proxysql-0
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-proxysql-0
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=proxysql
+ set +o xtrace
pod/upgrade-proxysql-proxysql-0 condition met
waiting for pod/upgrade-proxysql-proxysql-0 to become Ready.Ok
+ wait_for_running upgrade-proxysql-pxc 3
+ local name=upgrade-proxysql-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-0 480
+ local pod=upgrade-proxysql-pxc-0
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-0 condition met
waiting for pod/upgrade-proxysql-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-1 480
+ local pod=upgrade-proxysql-pxc-1
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-1
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-1 condition met
waiting for pod/upgrade-proxysql-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-2 480
+ local pod=upgrade-proxysql-pxc-2
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-2 condition met
waiting for pod/upgrade-proxysql-pxc-2 to become Ready.Ok
+ sleep 30
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xEGIMuK7ok
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zSZDLGYwhr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xEGIMuK7ok
++ cat /tmp/tmp.zSZDLGYwhr
++ rm /tmp/tmp.xEGIMuK7ok /tmp/tmp.zSZDLGYwhr
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
++ is_keyring_plugin_in_use upgrade-proxysql
++ local cluster=upgrade-proxysql
++ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
++ kubectl exec upgrade-proxysql-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ grep -E -o 'early-plugin-load=keyring_\w+.so'
++ return 1
+ [[ -n '' ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h upgrade-proxysql-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h upgrade-proxysql-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FxDeap7gge
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oYmCUMztCM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FxDeap7gge
++ cat /tmp/tmp.oYmCUMztCM
++ rm /tmp/tmp.FxDeap7gge /tmp/tmp.oYmCUMztCM
++ return 0
+ client_pod=pxc-client-6c975684f9-x5pqp
+ wait_pod pxc-client-6c975684f9-x5pqp
+ local pod=pxc-client-6c975684f9-x5pqp
+ local max_retry=480
+ local ns=
++ echo pxc-client-6c975684f9-x5pqp
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6c975684f9-x5pqp condition met
waiting for pod/pxc-client-6c975684f9-x5pqp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h upgrade-proxysql-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h upgrade-proxysql-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AwkyneqPDR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bkUOwUJhHE
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AwkyneqPDR
++ cat /tmp/tmp.bkUOwUJhHE
++ rm /tmp/tmp.AwkyneqPDR /tmp/tmp.bkUOwUJhHE
++ return 0
+ client_pod=pxc-client-6c975684f9-x5pqp
+ wait_pod pxc-client-6c975684f9-x5pqp
+ local pod=pxc-client-6c975684f9-x5pqp
+ local max_retry=480
+ local ns=
++ echo pxc-client-6c975684f9-x5pqp
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6c975684f9-x5pqp condition met
waiting for pod/pxc-client-6c975684f9-x5pqp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VapQuShiZj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.whYrbjIH8x
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VapQuShiZj
++ cat /tmp/tmp.whYrbjIH8x
++ rm /tmp/tmp.VapQuShiZj /tmp/tmp.whYrbjIH8x
++ return 0
+ client_pod=pxc-client-6c975684f9-x5pqp
+ wait_pod pxc-client-6c975684f9-x5pqp
+ local pod=pxc-client-6c975684f9-x5pqp
+ local max_retry=480
+ local ns=
++ echo pxc-client-6c975684f9-x5pqp
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6c975684f9-x5pqp condition met
waiting for pod/pxc-client-6c975684f9-x5pqp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.9XEC036CwX/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.9XEC036CwX/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.9XEC036CwX/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GaN0XRS1Vd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.c3if1KNPxC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GaN0XRS1Vd
++ cat /tmp/tmp.c3if1KNPxC
++ rm /tmp/tmp.GaN0XRS1Vd /tmp/tmp.c3if1KNPxC
++ return 0
+ client_pod=pxc-client-6c975684f9-x5pqp
+ wait_pod pxc-client-6c975684f9-x5pqp
+ local pod=pxc-client-6c975684f9-x5pqp
+ local max_retry=480
+ local ns=
++ echo pxc-client-6c975684f9-x5pqp
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6c975684f9-x5pqp condition met
waiting for pod/pxc-client-6c975684f9-x5pqp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.9XEC036CwX/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.9XEC036CwX/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.9XEC036CwX/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LSkAEtfAMk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DBRtJAKhvL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LSkAEtfAMk
++ cat /tmp/tmp.DBRtJAKhvL
++ rm /tmp/tmp.LSkAEtfAMk /tmp/tmp.DBRtJAKhvL
++ return 0
+ client_pod=pxc-client-6c975684f9-x5pqp
+ wait_pod pxc-client-6c975684f9-x5pqp
+ local pod=pxc-client-6c975684f9-x5pqp
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo pxc-client-6c975684f9-x5pqp
+ local container=
+ set +o xtrace
pod/pxc-client-6c975684f9-x5pqp condition met
waiting for pod/pxc-client-6c975684f9-x5pqp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.9XEC036CwX/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.9XEC036CwX/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.9XEC036CwX/select-1.sql
+ is_keyring_plugin_in_use upgrade-proxysql
+ local cluster=upgrade-proxysql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ grep -E -o 'early-plugin-load=keyring_\w+.so'
+ kubectl exec upgrade-proxysql-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
+ return 1
+ compare_generation 1 proxysql upgrade-proxysql
+ local generation=1
+ local proxy=proxysql
+ local cluster=upgrade-proxysql
+ local current_generation
+ [[ proxysql == \h\a\p\r\o\x\y ]]
+ containers=(pxc proxysql)
+ for container in '"${containers[@]}"'
+ check_generation 1 pxc upgrade-proxysql
+ local generation=1
+ local container=pxc
+ local cluster=upgrade-proxysql
+ local current_generation
++ kubectl_bin get statefulset upgrade-proxysql-pxc -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XjXTygG0k6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YV2jlnOQFy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset upgrade-proxysql-pxc -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XjXTygG0k6
++ cat /tmp/tmp.YV2jlnOQFy
++ rm /tmp/tmp.XjXTygG0k6 /tmp/tmp.YV2jlnOQFy
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ for container in '"${containers[@]}"'
+ check_generation 1 proxysql upgrade-proxysql
+ local generation=1
+ local container=proxysql
+ local cluster=upgrade-proxysql
+ local current_generation
++ kubectl_bin get statefulset upgrade-proxysql-proxysql -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.J3XDtgaHCF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9SuZwy3lOv
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset upgrade-proxysql-proxysql -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.J3XDtgaHCF
++ cat /tmp/tmp.9SuZwy3lOv
++ rm /tmp/tmp.J3XDtgaHCF /tmp/tmp.9SuZwy3lOv
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ desc 'upgrade operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
upgrade operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.kRsPfX9fH9
++ mktemp
+ local LAST_ERR=/tmp/tmp.1ZoJSNm3nX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.kRsPfX9fH9
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.1ZoJSNm3nX
+ rm /tmp/tmp.kRsPfX9fH9 /tmp/tmp.1ZoJSNm3nX
+ return 0
+ [[ -n pxc-operator ]]
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/deploy/cw-rbac.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.8ticNV8S3s
++ mktemp
+ local LAST_ERR=/tmp/tmp.Y4WuXITPFS
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.8ticNV8S3s
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.Y4WuXITPFS
+ rm /tmp/tmp.8ticNV8S3s /tmp/tmp.Y4WuXITPFS
+ return 0
+ kubectl_bin patch deployment percona-xtradb-cluster-operator '-p{"spec":{"template":{"spec":{"containers":[{"name":"percona-xtradb-cluster-operator","image":"perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd"}]}}}}' -n pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.JsLcfLjD1H
++ mktemp
+ local LAST_ERR=/tmp/tmp.kEKICaUiyq
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch deployment percona-xtradb-cluster-operator '-p{"spec":{"template":{"spec":{"containers":[{"name":"percona-xtradb-cluster-operator","image":"perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd"}]}}}}' -n pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.JsLcfLjD1H
deployment.apps/percona-xtradb-cluster-operator patched
+ cat /tmp/tmp.kEKICaUiyq
+ rm /tmp/tmp.JsLcfLjD1H /tmp/tmp.kEKICaUiyq
+ return 0
+ kubectl_bin rollout status deployment/percona-xtradb-cluster-operator -n pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.9sOwgs1NRw
++ mktemp
+ local LAST_ERR=/tmp/tmp.HF7nmf4wTW
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl rollout status deployment/percona-xtradb-cluster-operator -n pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.9sOwgs1NRw
Waiting for deployment "percona-xtradb-cluster-operator" rollout to finish: 0 of 1 updated replicas are available...
deployment "percona-xtradb-cluster-operator" successfully rolled out
+ cat /tmp/tmp.HF7nmf4wTW
+ rm /tmp/tmp.9sOwgs1NRw /tmp/tmp.HF7nmf4wTW
+ return 0
+ sleep 10
+ desc 'wait for operator upgrade'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for operator upgrade
-----------------------------------------------------------------------------------

+ local i=0
+ local max=60
++ grep -vc NAME
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image' -n pxc-operator
+++ mktemp
++ awk '{print $1}'
++ local LAST_OUT=/tmp/tmp.uhUuZdklnH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VR3eEtCLRp
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.uhUuZdklnH
++ cat /tmp/tmp.VR3eEtCLRp
++ rm /tmp/tmp.uhUuZdklnH /tmp/tmp.VR3eEtCLRp
++ return 0
+ [[ 1 -eq 1 ]]
+ '[' -n pxc-operator ']'
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mp39dgaVS4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.67BEpcg9AN
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.mp39dgaVS4
++ cat /tmp/tmp.67BEpcg9AN
++ rm /tmp/tmp.mp39dgaVS4 /tmp/tmp.67BEpcg9AN
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster5 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.tOtM1ZImm8
++ mktemp
+ local LAST_ERR=/tmp/tmp.JnRp28QzMG
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster5 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.tOtM1ZImm8
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster5" modified.
+ cat /tmp/tmp.JnRp28QzMG
+ rm /tmp/tmp.tOtM1ZImm8 /tmp/tmp.JnRp28QzMG
+ return 0
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image'
++ awk '{print $1}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wPTHdiWqxl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KC2W8HtztF
++ local exit_status=0
++ grep perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wPTHdiWqxl
++ cat /tmp/tmp.KC2W8HtztF
++ rm /tmp/tmp.wPTHdiWqxl /tmp/tmp.KC2W8HtztF
++ return 0
+ wait_pod percona-xtradb-cluster-operator-6cfdc6cb68-g98vt
+ local pod=percona-xtradb-cluster-operator-6cfdc6cb68-g98vt
+ local max_retry=480
+ local ns=
++ echo percona-xtradb-cluster-operator-6cfdc6cb68-g98vt
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-6cfdc6cb68-g98vt condition met
waiting for pod/percona-xtradb-cluster-operator-6cfdc6cb68-g98vt to become Ready.Ok
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pD4ZhmWlVl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vJBd7HSL2F
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pD4ZhmWlVl
++ cat /tmp/tmp.vJBd7HSL2F
++ rm /tmp/tmp.pD4ZhmWlVl /tmp/tmp.vJBd7HSL2F
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster5 --namespace=upgrade-proxysql-4143
++ mktemp
+ local LAST_OUT=/tmp/tmp.dpyIAXCJJW
++ mktemp
+ local LAST_ERR=/tmp/tmp.AY2leb1HJ2
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster5 --namespace=upgrade-proxysql-4143
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.dpyIAXCJJW
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster5" modified.
+ cat /tmp/tmp.AY2leb1HJ2
+ rm /tmp/tmp.dpyIAXCJJW /tmp/tmp.AY2leb1HJ2
+ return 0
+ desc 'check images and generation after operator upgrade'
+ set +o xtrace


-----------------------------------------------------------------------------------
check images and generation after operator upgrade
-----------------------------------------------------------------------------------

+ check_pxc_liveness upgrade-proxysql 3
+ local cluster=upgrade-proxysql
+ local cluster_size=3
+ wait_cluster_consistency upgrade-proxysql 3
+ local cluster_name=upgrade-proxysql
+ local cluster_size=3
+ local proxy_size=
+ '[' -z '' ']'
++ get_proxy_size upgrade-proxysql
++ local cluster=upgrade-proxysql
+++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.6SdJWWlhGu
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.akk3Qtvgwb
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.6SdJWWlhGu
+++ cat /tmp/tmp.akk3Qtvgwb
+++ rm /tmp/tmp.6SdJWWlhGu /tmp/tmp.akk3Qtvgwb
+++ return 0
++ [[ false == \t\r\u\e ]]
+++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kr72BL1Bzf
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.HaueiN91tX
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.kr72BL1Bzf
+++ cat /tmp/tmp.HaueiN91tX
+++ rm /tmp/tmp.kr72BL1Bzf /tmp/tmp.HaueiN91tX
+++ return 0
++ [[ true == \t\r\u\e ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.size}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JjgcA6UrgO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AdQWWNW76t
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.size}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.JjgcA6UrgO
++ cat /tmp/tmp.AdQWWNW76t
++ rm /tmp/tmp.JjgcA6UrgO /tmp/tmp.AdQWWNW76t
++ return 0
++ return
+ proxy_size=3
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=300
+ sleep 7
+ echo -n 'waiting for pxc/upgrade-proxysql to be ready'
waiting for pxc/upgrade-proxysql to be ready++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wd2g2myyzY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Nn6zyyb2T9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wd2g2myyzY
++ cat /tmp/tmp.Nn6zyyb2T9
++ rm /tmp/tmp.wd2g2myyzY /tmp/tmp.Nn6zyyb2T9
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LzwZ79jMap
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nYcpHU9NVI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LzwZ79jMap
++ cat /tmp/tmp.nYcpHU9NVI
++ rm /tmp/tmp.LzwZ79jMap /tmp/tmp.nYcpHU9NVI
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine upgrade-proxysql
+++ local cluster_name=upgrade-proxysql
++++ get_proxy upgrade-proxysql
++++ local target_cluster=upgrade-proxysql
+++++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.Fus4TlZ04S
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.Bd59JbQumO
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.Fus4TlZ04S
+++++ cat /tmp/tmp.Bd59JbQumO
+++++ rm /tmp/tmp.Fus4TlZ04S /tmp/tmp.Bd59JbQumO
+++++ return 0
++++ [[ false == \t\r\u\e ]]
+++++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.u8PoqInD25
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.4XdntMGYSq
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.u8PoqInD25
+++++ cat /tmp/tmp.4XdntMGYSq
+++++ rm /tmp/tmp.u8PoqInD25 /tmp/tmp.4XdntMGYSq
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo upgrade-proxysql-proxysql
++++ return
+++ local cluster_proxy=upgrade-proxysql-proxysql
+++ echo proxysql
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KiVmV5DmN2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AHd8WM3sa5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KiVmV5DmN2
++ cat /tmp/tmp.AHd8WM3sa5
++ rm /tmp/tmp.KiVmV5DmN2 /tmp/tmp.AHd8WM3sa5
++ return 0
+ [[ 3 == \3 ]]
+ echo

+ wait_for_running upgrade-proxysql-pxc 3
+ local name=upgrade-proxysql-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-0 480
+ local pod=upgrade-proxysql-pxc-0
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-0
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-0 condition met
waiting for pod/upgrade-proxysql-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-1 480
+ local pod=upgrade-proxysql-pxc-1
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-1 condition met
waiting for pod/upgrade-proxysql-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-2 480
+ local pod=upgrade-proxysql-pxc-2
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-2 condition met
waiting for pod/upgrade-proxysql-pxc-2 to become Ready.Ok
++ seq 0 2
+ for i in '$(seq 0 $((cluster_size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YdvvWFvTre
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bEG3QCnRT4
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YdvvWFvTre
++ cat /tmp/tmp.bEG3QCnRT4
++ rm /tmp/tmp.YdvvWFvTre /tmp/tmp.bEG3QCnRT4
++ return 0
+ client_pod=pxc-client-6c975684f9-x5pqp
+ wait_pod pxc-client-6c975684f9-x5pqp
+ local pod=pxc-client-6c975684f9-x5pqp
+ local max_retry=480
+ local ns=
++ echo pxc-client-6c975684f9-x5pqp
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6c975684f9-x5pqp condition met
waiting for pod/pxc-client-6c975684f9-x5pqp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.9XEC036CwX/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.9XEC036CwX/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.9XEC036CwX/select-1.sql
+ for i in '$(seq 0 $((cluster_size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Obkb8TMyQ5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AKUOlmZo7c
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Obkb8TMyQ5
++ cat /tmp/tmp.AKUOlmZo7c
++ rm /tmp/tmp.Obkb8TMyQ5 /tmp/tmp.AKUOlmZo7c
++ return 0
+ client_pod=pxc-client-6c975684f9-x5pqp
+ wait_pod pxc-client-6c975684f9-x5pqp
+ local pod=pxc-client-6c975684f9-x5pqp
+ local max_retry=480
+ local ns=
++ echo pxc-client-6c975684f9-x5pqp
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6c975684f9-x5pqp condition met
waiting for pod/pxc-client-6c975684f9-x5pqp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.9XEC036CwX/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.9XEC036CwX/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.9XEC036CwX/select-1.sql
+ for i in '$(seq 0 $((cluster_size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wTpxzXDrgv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.p3pLdeN9GR
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wTpxzXDrgv
++ cat /tmp/tmp.p3pLdeN9GR
++ rm /tmp/tmp.wTpxzXDrgv /tmp/tmp.p3pLdeN9GR
++ return 0
+ client_pod=pxc-client-6c975684f9-x5pqp
+ wait_pod pxc-client-6c975684f9-x5pqp
+ local pod=pxc-client-6c975684f9-x5pqp
+ local max_retry=480
+ local ns=
++ echo pxc-client-6c975684f9-x5pqp
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6c975684f9-x5pqp condition met
waiting for pod/pxc-client-6c975684f9-x5pqp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.9XEC036CwX/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.9XEC036CwX/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.9XEC036CwX/select-1.sql
++ kubectl_bin get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vP99MRUibl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mfFjrnd0ZV
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vP99MRUibl
++ cat /tmp/tmp.mfFjrnd0ZV
++ rm /tmp/tmp.vP99MRUibl /tmp/tmp.mfFjrnd0ZV
++ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd == perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.56X5wqmRJU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QoEO5Qkc4a
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.56X5wqmRJU
++ cat /tmp/tmp.QoEO5Qkc4a
++ rm /tmp/tmp.56X5wqmRJU /tmp/tmp.QoEO5Qkc4a
++ return 0
+ [[ percona/proxysql2:2.7.3-1.2 == percona/proxysql2:2.7.3-1.2 ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E9h1o5unBF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hWpx2wYKQk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.E9h1o5unBF
++ cat /tmp/tmp.hWpx2wYKQk
++ rm /tmp/tmp.E9h1o5unBF /tmp/tmp.hWpx2wYKQk
++ return 0
+ [[ percona/haproxy:2.8.17 == percona/haproxy:2.8.17 ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.backup.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oFL7EmtWkD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NCkVNXYbiP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.backup.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.oFL7EmtWkD
++ cat /tmp/tmp.NCkVNXYbiP
++ rm /tmp/tmp.oFL7EmtWkD /tmp/tmp.NCkVNXYbiP
++ return 0
+ [[ percona/percona-xtrabackup:2.4.29 == percona/percona-xtrabackup:2.4.29 ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.pmm.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lO3rHKOlce
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5PjIenJTSM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.pmm.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.lO3rHKOlce
++ cat /tmp/tmp.5PjIenJTSM
++ rm /tmp/tmp.lO3rHKOlce /tmp/tmp.5PjIenJTSM
++ return 0
+ [[ percona/pmm-client:3.5.0 == percona/pmm-client:3.5.0 ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.logcollector.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TJLgtINlcG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7SCg53nubU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.logcollector.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TJLgtINlcG
++ cat /tmp/tmp.7SCg53nubU
++ rm /tmp/tmp.TJLgtINlcG /tmp/tmp.7SCg53nubU
++ return 0
+ [[ percona/fluentbit:4.0.1-1 == percona/fluentbit:4.0.1-1 ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.pxc.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9IOSA4xe1f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xZLGZkdGkI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.pxc.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.9IOSA4xe1f
++ cat /tmp/tmp.xZLGZkdGkI
++ rm /tmp/tmp.9IOSA4xe1f /tmp/tmp.xZLGZkdGkI
++ return 0
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 == percona/percona-xtradb-cluster:5.7.44-31.65 ]]
+ : Operator image has been updated correctly
+ compare_generation 1 proxysql upgrade-proxysql
+ local generation=1
+ local proxy=proxysql
+ local cluster=upgrade-proxysql
+ local current_generation
+ [[ proxysql == \h\a\p\r\o\x\y ]]
+ containers=(pxc proxysql)
+ for container in '"${containers[@]}"'
+ check_generation 1 pxc upgrade-proxysql
+ local generation=1
+ local container=pxc
+ local cluster=upgrade-proxysql
+ local current_generation
++ kubectl_bin get statefulset upgrade-proxysql-pxc -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VEF93KgTcG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.e3LV57wQj1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset upgrade-proxysql-pxc -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VEF93KgTcG
++ cat /tmp/tmp.e3LV57wQj1
++ rm /tmp/tmp.VEF93KgTcG /tmp/tmp.e3LV57wQj1
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ for container in '"${containers[@]}"'
+ check_generation 1 proxysql upgrade-proxysql
+ local generation=1
+ local container=proxysql
+ local cluster=upgrade-proxysql
+ local current_generation
++ kubectl_bin get statefulset upgrade-proxysql-proxysql -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4gT1Its6pE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QydsztY2NM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset upgrade-proxysql-proxysql -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4gT1Its6pE
++ cat /tmp/tmp.QydsztY2NM
++ rm /tmp/tmp.4gT1Its6pE /tmp/tmp.QydsztY2NM
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ desc 'patch pxc images and upgrade'
+ set +o xtrace


-----------------------------------------------------------------------------------
patch pxc images and upgrade
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc upgrade-proxysql --type=merge --patch '{
        "spec": {
            "crVersion": "1.20.0",
            "pxc": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc5.7" },
            "pmm": { "image": "perconalab/pmm-client:dev-latest" },
            "haproxy": { "image": "perconalab/percona-xtradb-cluster-operator:main-haproxy" },
            "proxysql": { "image": "perconalab/percona-xtradb-cluster-operator:main-proxysql" },
            "logcollector": { "image": "perconalab/fluentbit:main-logcollector" },
            "backup": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup" }
        }}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.cmXFAATxoG
++ mktemp
+ local LAST_ERR=/tmp/tmp.4kYwu8gCPh
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc upgrade-proxysql --type=merge --patch '{
        "spec": {
            "crVersion": "1.20.0",
            "pxc": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc5.7" },
            "pmm": { "image": "perconalab/pmm-client:dev-latest" },
            "haproxy": { "image": "perconalab/percona-xtradb-cluster-operator:main-haproxy" },
            "proxysql": { "image": "perconalab/percona-xtradb-cluster-operator:main-proxysql" },
            "logcollector": { "image": "perconalab/fluentbit:main-logcollector" },
            "backup": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup" }
        }}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.cmXFAATxoG
perconaxtradbcluster.pxc.percona.com/upgrade-proxysql patched
+ cat /tmp/tmp.4kYwu8gCPh
+ rm /tmp/tmp.cmXFAATxoG /tmp/tmp.4kYwu8gCPh
+ return 0
+ sleep 10
+ desc 'check images and generation after full upgrade'
+ set +o xtrace


-----------------------------------------------------------------------------------
check images and generation after full upgrade
-----------------------------------------------------------------------------------

+ check_pxc_liveness upgrade-proxysql 3
+ local cluster=upgrade-proxysql
+ local cluster_size=3
+ wait_cluster_consistency upgrade-proxysql 3
+ local cluster_name=upgrade-proxysql
+ local cluster_size=3
+ local proxy_size=
+ '[' -z '' ']'
++ get_proxy_size upgrade-proxysql
++ local cluster=upgrade-proxysql
+++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.mIGqoopXP3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Qs8LhutIst
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.mIGqoopXP3
+++ cat /tmp/tmp.Qs8LhutIst
+++ rm /tmp/tmp.mIGqoopXP3 /tmp/tmp.Qs8LhutIst
+++ return 0
++ [[ false == \t\r\u\e ]]
+++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SnOmmcNVzN
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.MupQG2GkSA
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.SnOmmcNVzN
+++ cat /tmp/tmp.MupQG2GkSA
+++ rm /tmp/tmp.SnOmmcNVzN /tmp/tmp.MupQG2GkSA
+++ return 0
++ [[ true == \t\r\u\e ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.size}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.h303cyn6M7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WPQKNR0sFT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.size}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.h303cyn6M7
++ cat /tmp/tmp.WPQKNR0sFT
++ rm /tmp/tmp.h303cyn6M7 /tmp/tmp.WPQKNR0sFT
++ return 0
++ return
+ proxy_size=3
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=300
+ sleep 7
+ echo -n 'waiting for pxc/upgrade-proxysql to be ready'
waiting for pxc/upgrade-proxysql to be ready++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Xa9MeiDCFT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SASR969anM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Xa9MeiDCFT
++ cat /tmp/tmp.SASR969anM
++ rm /tmp/tmp.Xa9MeiDCFT /tmp/tmp.SASR969anM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 0 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GawCzpzwbJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.leemlu8WFT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GawCzpzwbJ
++ cat /tmp/tmp.leemlu8WFT
++ rm /tmp/tmp.GawCzpzwbJ /tmp/tmp.leemlu8WFT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 1 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6L2YvhigB5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MYuaTuTAVO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6L2YvhigB5
++ cat /tmp/tmp.MYuaTuTAVO
++ rm /tmp/tmp.6L2YvhigB5 /tmp/tmp.MYuaTuTAVO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 2 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1PwLSTRpFl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XQ5OW3dHCl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1PwLSTRpFl
++ cat /tmp/tmp.XQ5OW3dHCl
++ rm /tmp/tmp.1PwLSTRpFl /tmp/tmp.XQ5OW3dHCl
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 3 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NlI4iNe9eC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.n83yTNt1Yd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NlI4iNe9eC
++ cat /tmp/tmp.n83yTNt1Yd
++ rm /tmp/tmp.NlI4iNe9eC /tmp/tmp.n83yTNt1Yd
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 4 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QZx2rmTd28
+++ mktemp
++ local LAST_ERR=/tmp/tmp.887sxf4ZUY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.QZx2rmTd28
++ cat /tmp/tmp.887sxf4ZUY
++ rm /tmp/tmp.QZx2rmTd28 /tmp/tmp.887sxf4ZUY
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 5 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SD4iCeTJxB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.D2ko1e5D4q
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SD4iCeTJxB
++ cat /tmp/tmp.D2ko1e5D4q
++ rm /tmp/tmp.SD4iCeTJxB /tmp/tmp.D2ko1e5D4q
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 6 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uW3EOCVpws
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aLUMFjqyq7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.uW3EOCVpws
++ cat /tmp/tmp.aLUMFjqyq7
++ rm /tmp/tmp.uW3EOCVpws /tmp/tmp.aLUMFjqyq7
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 7 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LMtmvLbTY7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o1UC1Hf7hq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LMtmvLbTY7
++ cat /tmp/tmp.o1UC1Hf7hq
++ rm /tmp/tmp.LMtmvLbTY7 /tmp/tmp.o1UC1Hf7hq
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 8 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2u0QnwJtQm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qsXBEOPfa5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2u0QnwJtQm
++ cat /tmp/tmp.qsXBEOPfa5
++ rm /tmp/tmp.2u0QnwJtQm /tmp/tmp.qsXBEOPfa5
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 9 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4H5MOKgPJO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.a90om5jsD6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4H5MOKgPJO
++ cat /tmp/tmp.a90om5jsD6
++ rm /tmp/tmp.4H5MOKgPJO /tmp/tmp.a90om5jsD6
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 10 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NujNNFrdOn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QIbE9TjHVC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NujNNFrdOn
++ cat /tmp/tmp.QIbE9TjHVC
++ rm /tmp/tmp.NujNNFrdOn /tmp/tmp.QIbE9TjHVC
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 11 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3Q3Ksr7HYv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pyWOwysYBo
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3Q3Ksr7HYv
++ cat /tmp/tmp.pyWOwysYBo
++ rm /tmp/tmp.3Q3Ksr7HYv /tmp/tmp.pyWOwysYBo
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 12 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vOYoatDHxi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9XgylGp9EF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vOYoatDHxi
++ cat /tmp/tmp.9XgylGp9EF
++ rm /tmp/tmp.vOYoatDHxi /tmp/tmp.9XgylGp9EF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 13 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Xqa9LSp37V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ywKhUCFGZI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Xqa9LSp37V
++ cat /tmp/tmp.ywKhUCFGZI
++ rm /tmp/tmp.Xqa9LSp37V /tmp/tmp.ywKhUCFGZI
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 14 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wqlRE9Lty7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.61U2TuC3eg
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wqlRE9Lty7
++ cat /tmp/tmp.61U2TuC3eg
++ rm /tmp/tmp.wqlRE9Lty7 /tmp/tmp.61U2TuC3eg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 15 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aXusvWYBtX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v4m8RX3ei2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.aXusvWYBtX
++ cat /tmp/tmp.v4m8RX3ei2
++ rm /tmp/tmp.aXusvWYBtX /tmp/tmp.v4m8RX3ei2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 16 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dU5HW4uVtr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iLdNH42VyJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.dU5HW4uVtr
++ cat /tmp/tmp.iLdNH42VyJ
++ rm /tmp/tmp.dU5HW4uVtr /tmp/tmp.iLdNH42VyJ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 17 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tkOijYSvbK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EGyA7341EE
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.tkOijYSvbK
++ cat /tmp/tmp.EGyA7341EE
++ rm /tmp/tmp.tkOijYSvbK /tmp/tmp.EGyA7341EE
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 18 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FHht3AfNFv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XyQ1vPURQq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FHht3AfNFv
++ cat /tmp/tmp.XyQ1vPURQq
++ rm /tmp/tmp.FHht3AfNFv /tmp/tmp.XyQ1vPURQq
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 19 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ndpLoJwJh7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lgYrnU17ET
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ndpLoJwJh7
++ cat /tmp/tmp.lgYrnU17ET
++ rm /tmp/tmp.ndpLoJwJh7 /tmp/tmp.lgYrnU17ET
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 20 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YdMSicQhao
+++ mktemp
++ local LAST_ERR=/tmp/tmp.29spkeY8OW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YdMSicQhao
++ cat /tmp/tmp.29spkeY8OW
++ rm /tmp/tmp.YdMSicQhao /tmp/tmp.29spkeY8OW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 21 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6AZdd2tscB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.diLcECPPTx
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6AZdd2tscB
++ cat /tmp/tmp.diLcECPPTx
++ rm /tmp/tmp.6AZdd2tscB /tmp/tmp.diLcECPPTx
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 22 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IscN96V9tW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sxJLqxfWQn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IscN96V9tW
++ cat /tmp/tmp.sxJLqxfWQn
++ rm /tmp/tmp.IscN96V9tW /tmp/tmp.sxJLqxfWQn
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 23 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1w3F5Dz0i8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gG3BCzI1ym
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1w3F5Dz0i8
++ cat /tmp/tmp.gG3BCzI1ym
++ rm /tmp/tmp.1w3F5Dz0i8 /tmp/tmp.gG3BCzI1ym
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 24 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LPGDMMQDDe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gRODzUDnHl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LPGDMMQDDe
++ cat /tmp/tmp.gRODzUDnHl
++ rm /tmp/tmp.LPGDMMQDDe /tmp/tmp.gRODzUDnHl
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 25 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nl34kFwhu2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.I5AHxFFd4e
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nl34kFwhu2
++ cat /tmp/tmp.I5AHxFFd4e
++ rm /tmp/tmp.nl34kFwhu2 /tmp/tmp.I5AHxFFd4e
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 26 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1yAKKU0Y7r
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OcVb3aBTuY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1yAKKU0Y7r
++ cat /tmp/tmp.OcVb3aBTuY
++ rm /tmp/tmp.1yAKKU0Y7r /tmp/tmp.OcVb3aBTuY
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 27 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ofiJB266YS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ipFbWozLt7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ofiJB266YS
++ cat /tmp/tmp.ipFbWozLt7
++ rm /tmp/tmp.ofiJB266YS /tmp/tmp.ipFbWozLt7
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 28 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ko1jCdGz2d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BMFICUtFbQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ko1jCdGz2d
++ cat /tmp/tmp.BMFICUtFbQ
++ rm /tmp/tmp.ko1jCdGz2d /tmp/tmp.BMFICUtFbQ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 29 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.m4QNIeynSd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VxzS6BiUti
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.m4QNIeynSd
++ cat /tmp/tmp.VxzS6BiUti
++ rm /tmp/tmp.m4QNIeynSd /tmp/tmp.VxzS6BiUti
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ye2JypYjqc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kBT8bUhcD8
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ye2JypYjqc
++ cat /tmp/tmp.kBT8bUhcD8
++ rm /tmp/tmp.ye2JypYjqc /tmp/tmp.kBT8bUhcD8
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine upgrade-proxysql
+++ local cluster_name=upgrade-proxysql
++++ get_proxy upgrade-proxysql
++++ local target_cluster=upgrade-proxysql
+++++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.QhzSmOudMp
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.v15ZbXEJYH
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.QhzSmOudMp
+++++ cat /tmp/tmp.v15ZbXEJYH
+++++ rm /tmp/tmp.QhzSmOudMp /tmp/tmp.v15ZbXEJYH
+++++ return 0
++++ [[ false == \t\r\u\e ]]
+++++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.lPKvmoyrJt
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.6bLaZ6XTi6
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.lPKvmoyrJt
+++++ cat /tmp/tmp.6bLaZ6XTi6
+++++ rm /tmp/tmp.lPKvmoyrJt /tmp/tmp.6bLaZ6XTi6
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo upgrade-proxysql-proxysql
++++ return
+++ local cluster_proxy=upgrade-proxysql-proxysql
+++ echo proxysql
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cnMwzXXB3a
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6ZwO0BgtLj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.cnMwzXXB3a
++ cat /tmp/tmp.6ZwO0BgtLj
++ rm /tmp/tmp.cnMwzXXB3a /tmp/tmp.6ZwO0BgtLj
++ return 0
+ [[ 3 == \3 ]]
+ echo

+ wait_for_running upgrade-proxysql-pxc 3
+ local name=upgrade-proxysql-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-0 480
+ local pod=upgrade-proxysql-pxc-0
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-0 condition met
waiting for pod/upgrade-proxysql-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-1 480
+ local pod=upgrade-proxysql-pxc-1
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-1 condition met
waiting for pod/upgrade-proxysql-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-2 480
+ local pod=upgrade-proxysql-pxc-2
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-2 condition met
waiting for pod/upgrade-proxysql-pxc-2 to become Ready.Ok
++ seq 0 2
+ for i in '$(seq 0 $((cluster_size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Uk0comhG96
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5uqMcpkxTT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Uk0comhG96
++ cat /tmp/tmp.5uqMcpkxTT
++ rm /tmp/tmp.Uk0comhG96 /tmp/tmp.5uqMcpkxTT
++ return 0
+ client_pod=pxc-client-6c975684f9-x5pqp
+ wait_pod pxc-client-6c975684f9-x5pqp
+ local pod=pxc-client-6c975684f9-x5pqp
+ local max_retry=480
+ local ns=
++ echo pxc-client-6c975684f9-x5pqp
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6c975684f9-x5pqp condition met
waiting for pod/pxc-client-6c975684f9-x5pqp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.9XEC036CwX/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.9XEC036CwX/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.9XEC036CwX/select-1.sql
+ for i in '$(seq 0 $((cluster_size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NNklKvNRI6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d0qNro7HEc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NNklKvNRI6
++ cat /tmp/tmp.d0qNro7HEc
++ rm /tmp/tmp.NNklKvNRI6 /tmp/tmp.d0qNro7HEc
++ return 0
+ client_pod=pxc-client-6c975684f9-x5pqp
+ wait_pod pxc-client-6c975684f9-x5pqp
+ local pod=pxc-client-6c975684f9-x5pqp
+ local max_retry=480
+ local ns=
++ echo pxc-client-6c975684f9-x5pqp
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6c975684f9-x5pqp condition met
waiting for pod/pxc-client-6c975684f9-x5pqp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.9XEC036CwX/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.9XEC036CwX/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.9XEC036CwX/select-1.sql
+ for i in '$(seq 0 $((cluster_size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G3BVqHrAdx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.a4DaCSRN60
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.G3BVqHrAdx
++ cat /tmp/tmp.a4DaCSRN60
++ rm /tmp/tmp.G3BVqHrAdx /tmp/tmp.a4DaCSRN60
++ return 0
+ client_pod=pxc-client-6c975684f9-x5pqp
+ wait_pod pxc-client-6c975684f9-x5pqp
+ local pod=pxc-client-6c975684f9-x5pqp
+ local max_retry=480
+ local ns=
++ echo pxc-client-6c975684f9-x5pqp
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6c975684f9-x5pqp condition met
waiting for pod/pxc-client-6c975684f9-x5pqp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.9XEC036CwX/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.9XEC036CwX/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.9XEC036CwX/select-1.sql
++ kubectl_bin get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XVYsBV5zdf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Xlzd8P7RsF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XVYsBV5zdf
++ cat /tmp/tmp.Xlzd8P7RsF
++ rm /tmp/tmp.XVYsBV5zdf /tmp/tmp.Xlzd8P7RsF
++ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd == perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.taaHlTxJxo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kpUlhJzAEO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.taaHlTxJxo
++ cat /tmp/tmp.kpUlhJzAEO
++ rm /tmp/tmp.taaHlTxJxo /tmp/tmp.kpUlhJzAEO
++ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-proxysql == perconalab/percona-xtradb-cluster-operator:main-proxysql ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QcksMPlnAW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fXWpUyxHkb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.QcksMPlnAW
++ cat /tmp/tmp.fXWpUyxHkb
++ rm /tmp/tmp.QcksMPlnAW /tmp/tmp.fXWpUyxHkb
++ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-haproxy == perconalab/percona-xtradb-cluster-operator:main-haproxy ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.backup.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fEUVbxKQNZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jDJHcy3lGI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.backup.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.fEUVbxKQNZ
++ cat /tmp/tmp.jDJHcy3lGI
++ rm /tmp/tmp.fEUVbxKQNZ /tmp/tmp.jDJHcy3lGI
++ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup == perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.pmm.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0vaeRn5gaj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jZD9Zlwm3R
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.pmm.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0vaeRn5gaj
++ cat /tmp/tmp.jZD9Zlwm3R
++ rm /tmp/tmp.0vaeRn5gaj /tmp/tmp.jZD9Zlwm3R
++ return 0
+ [[ perconalab/pmm-client:dev-latest == perconalab/pmm-client:dev-latest ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.logcollector.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SQDYrN1997
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KexToq3rdM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.logcollector.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SQDYrN1997
++ cat /tmp/tmp.KexToq3rdM
++ rm /tmp/tmp.SQDYrN1997 /tmp/tmp.KexToq3rdM
++ return 0
+ [[ perconalab/fluentbit:main-logcollector == perconalab/fluentbit:main-logcollector ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.pxc.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.B1r9OxtYLg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OILRRLINRC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.pxc.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.B1r9OxtYLg
++ cat /tmp/tmp.OILRRLINRC
++ rm /tmp/tmp.B1r9OxtYLg /tmp/tmp.OILRRLINRC
++ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 == perconalab/percona-xtradb-cluster-operator:main-pxc5.7 ]]
+ : Cluster images have been updated correctly
+ compare_generation 2 proxysql upgrade-proxysql
+ local generation=2
+ local proxy=proxysql
+ local cluster=upgrade-proxysql
+ local current_generation
+ [[ proxysql == \h\a\p\r\o\x\y ]]
+ containers=(pxc proxysql)
+ for container in '"${containers[@]}"'
+ check_generation 2 pxc upgrade-proxysql
+ local generation=2
+ local container=pxc
+ local cluster=upgrade-proxysql
+ local current_generation
++ kubectl_bin get statefulset upgrade-proxysql-pxc -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VyKKdAryTr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cFLhiv60ET
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset upgrade-proxysql-pxc -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VyKKdAryTr
++ cat /tmp/tmp.cFLhiv60ET
++ rm /tmp/tmp.VyKKdAryTr /tmp/tmp.cFLhiv60ET
++ return 0
+ current_generation=2
+ [[ 2 != \2 ]]
+ for container in '"${containers[@]}"'
+ check_generation 2 proxysql upgrade-proxysql
+ local generation=2
+ local container=proxysql
+ local cluster=upgrade-proxysql
+ local current_generation
++ kubectl_bin get statefulset upgrade-proxysql-proxysql -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VWP3OwP6Q4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eltyzs1nJT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset upgrade-proxysql-proxysql -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VWP3OwP6Q4
++ cat /tmp/tmp.eltyzs1nJT
++ rm /tmp/tmp.VWP3OwP6Q4 /tmp/tmp.eltyzs1nJT
++ return 0
+ current_generation=2
+ [[ 2 != \2 ]]
+ compare_kubectl statefulset/upgrade-proxysql-pxc
+ local resource=statefulset/upgrade-proxysql-pxc
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc.yml
+ local new_result=/tmp/tmp.9XEC036CwX/statefulset_upgrade-proxysql-pxc.yml
+ desc 'compare statefulset/upgrade-proxysql-pxc-'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/upgrade-proxysql-pxc-
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-eks.yml ']'
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ version_gt 1.33
++ echo '1.32 >= 1.33'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.29
++ echo '1.32 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-k129.yml ']'
+ version_gt 1.27
++ echo '1.32 >= 1.27'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-k127.yml ']'
+ version_gt 1.24
++ echo '1.32 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-k124.yml ']'
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-k122.yml ']'
+ version_gt 1.21
++ echo '1.32 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-oc.yml ']'
+ version_gt 1.29
++ echo '1.32 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-k129-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-eks.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-aks.yml ']'
+ kubectl_bin get -o yaml statefulset/upgrade-proxysql-pxc
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.labels."batch.kubernetes.io/job-name") |
			del(.metadata.labels."job-name") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") |
			del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") |
			del(.spec.template.metadata.labels."job-name") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-proxysql-4143", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.spec.crVersion) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.wWJ69TW7jc
++ mktemp
+ local LAST_ERR=/tmp/tmp.sHp5Zr5yvy
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/upgrade-proxysql-pxc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.wWJ69TW7jc
+ cat /tmp/tmp.sHp5Zr5yvy
+ rm /tmp/tmp.wWJ69TW7jc /tmp/tmp.sHp5Zr5yvy
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc.yml /tmp/tmp.9XEC036CwX/statefulset_upgrade-proxysql-pxc.yml
+ log 'compare_kubectl: statefulset/upgrade-proxysql-pxc OK'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-08T19:01:55+0000]' compare_kubectl: statefulset/upgrade-proxysql-pxc OK
[2026-04-08T19:01:55+0000] compare_kubectl: statefulset/upgrade-proxysql-pxc OK
+ compare_kubectl statefulset/upgrade-proxysql-proxysql
+ local resource=statefulset/upgrade-proxysql-proxysql
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql.yml
+ local new_result=/tmp/tmp.9XEC036CwX/statefulset_upgrade-proxysql-proxysql.yml
+ desc 'compare statefulset/upgrade-proxysql-proxysql-'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/upgrade-proxysql-proxysql-
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-eks.yml ']'
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ version_gt 1.33
++ echo '1.32 >= 1.33'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.29
++ echo '1.32 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-k129.yml ']'
+ version_gt 1.27
++ echo '1.32 >= 1.27'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-k127.yml ']'
+ version_gt 1.24
++ echo '1.32 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-k124.yml ']'
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-k122.yml ']'
+ version_gt 1.21
++ echo '1.32 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-oc.yml ']'
+ version_gt 1.29
++ echo '1.32 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-k129-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-eks.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-aks.yml ']'
+ kubectl_bin get -o yaml statefulset/upgrade-proxysql-proxysql
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.labels."batch.kubernetes.io/job-name") |
			del(.metadata.labels."job-name") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") |
			del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") |
			del(.spec.template.metadata.labels."job-name") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-proxysql-4143", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.spec.crVersion) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.m2vTkzH9BT
++ mktemp
+ local LAST_ERR=/tmp/tmp.Z9RavLJFWL
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/upgrade-proxysql-proxysql
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.m2vTkzH9BT
+ cat /tmp/tmp.Z9RavLJFWL
+ rm /tmp/tmp.m2vTkzH9BT /tmp/tmp.Z9RavLJFWL
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql.yml /tmp/tmp.9XEC036CwX/statefulset_upgrade-proxysql-proxysql.yml
+ log 'compare_kubectl: statefulset/upgrade-proxysql-proxysql OK'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-08T19:01:56+0000]' compare_kubectl: statefulset/upgrade-proxysql-proxysql OK
[2026-04-08T19:01:56+0000] compare_kubectl: statefulset/upgrade-proxysql-proxysql OK
+ desc cleanup
+ set +o xtrace


-----------------------------------------------------------------------------------
cleanup
-----------------------------------------------------------------------------------

+ destroy upgrade-proxysql-4143
+ local namespace=upgrade-proxysql-4143
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
+ grep -v level=info
+ grep -v 'the object has been modified'
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
++ get_operator_pod
+ grep -v 'get backup status: Job.batch'
++ local label_prefix=app.kubernetes.io/
+ sort -u
+ tee /tmp/tmp.9XEC036CwX/operator.log
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ head -1
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VcCMbags8D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UmhJHScIwU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VcCMbags8D
++ cat /tmp/tmp.UmhJHScIwU
++ rm /tmp/tmp.VcCMbags8D /tmp/tmp.UmhJHScIwU
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-6cfdc6cb68-g98vt
++ mktemp
+ local LAST_OUT=/tmp/tmp.783UrNBih1
++ mktemp
+ local LAST_ERR=/tmp/tmp.2S6dhbNng1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-6cfdc6cb68-g98vt
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.783UrNBih1
+ cat /tmp/tmp.2S6dhbNng1
+ rm /tmp/tmp.783UrNBih1 /tmp/tmp.2S6dhbNng1
+ return 0
2026-04-08T18:56:24.197Z	INFO	setup	Feature gates	{"PXCO_FEATURE_GATES": "", "enabled": ""}
2026-04-08T18:56:24.197Z	INFO	setup	Manager starting up	{"gitCommit": "b77708ddec65628a6739d09dda31398e5af42470", "gitBranch": "PR-2428-b77708dd", "buildTime": "2026-04-08T15:05:07Z", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"}
2026-04-08T18:56:24.197Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.32.13-gke.1205000"}
2026-04-08T18:56:24.200Z	INFO	setup	Registering Components.
2026-04-08T18:56:24.978Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2026-04-08T18:56:24.978Z	INFO	controller-runtime.metrics	Starting metrics server
2026-04-08T18:56:24.978Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2026-04-08T18:56:24.978Z	INFO	controller-runtime.webhook	Starting webhook server
2026-04-08T18:56:24.978Z	INFO	setup	Starting the Cmd.
2026-04-08T18:56:24.978Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2026-04-08T18:56:24.979Z	INFO	controller-runtime.certwatcher	Starting certificate poll+watcher	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"}
2026-04-08T18:56:24.979Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"}
2026-04-08T18:56:24.979Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2026-04-08T18:56:25.079Z	INFO	Attempting to acquire leader lease...	{"lock": "pxc-operator/08db1feb.percona.com"}
2026-04-08T18:56:42.703Z	DEBUG	events	percona-xtradb-cluster-operator-6cfdc6cb68-g98vt_6baccb2e-c96b-44c9-96c5-74b4b36f7ed1 became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"7a3b92a4-ca0e-40fd-995e-0cacfb650df7","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1775674602698111009"}, "reason": "LeaderElection"}
2026-04-08T18:56:42.703Z	INFO	Starting EventSource	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"}
2026-04-08T18:56:42.703Z	INFO	Starting EventSource	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"}
2026-04-08T18:56:42.703Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2026-04-08T18:56:42.703Z	INFO	Successfully acquired lease	{"lock": "pxc-operator/08db1feb.percona.com"}
2026-04-08T18:56:42.704Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2026-04-08T18:56:42.804Z	INFO	Starting Controller	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"}
2026-04-08T18:56:42.804Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1}
2026-04-08T18:56:42.904Z	INFO	Starting Controller	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"}
2026-04-08T18:56:42.904Z	INFO	Starting workers	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1}
2026-04-08T18:56:42.905Z	INFO	Starting Controller	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"}
2026-04-08T18:56:42.905Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1}
2026-04-08T18:56:44.369Z	DEBUG	Updating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "3136d41b-bc2e-41b1-b45c-f8bb2fe8a76a", "object": "upgrade-proxysql-proxysql", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}", "hashChanged": true, "metaChanged": true}
2026-04-08T18:56:44.461Z	INFO	Creating or updating backup job	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "3136d41b-bc2e-41b1-b45c-f8bb2fe8a76a", "name": "d96a3-daily-backup", "schedule": "0 0 * * *"}
2026-04-08T18:56:45.028Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "3136d41b-bc2e-41b1-b45c-f8bb2fe8a76a"}
2026-04-08T18:56:51.753Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "5d687194-5de8-44d2-bc7c-6fcf520ee296"}
2026-04-08T18:56:57.932Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "9d1c3c76-c7ec-4eeb-b9c9-caf0986c2f80"}
2026-04-08T18:57:03.931Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "66a456db-5f5f-4377-8254-4a2c9f697432"}
2026-04-08T18:57:10.321Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "e230db60-194b-4e7b-9618-d71014f68f08"}
2026-04-08T18:57:16.138Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "39115ee3-7a21-4556-88e6-28ee9cf44ad7"}
2026-04-08T18:57:23.107Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "a4c374aa-88f6-4fb7-820a-0f41e96c0eb0"}
2026-04-08T18:57:28.777Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "359a49d5-024d-4cba-bc0f-9ef67a6ff75f"}
2026-04-08T18:57:34.767Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "0778455c-7d99-45dd-91b5-d7b98ccf482d"}
2026-04-08T18:57:41.387Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "b5eb9e09-a26c-4447-ac58-58a4de3a663e"}
2026-04-08T18:57:47.657Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "e8ab510c-65ab-428f-b2a4-07154f7a69f4"}
2026-04-08T18:57:52.988Z	DEBUG	Updating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "object": "upgrade-proxysql-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2026-04-08T18:57:53.049Z	DEBUG	Updating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "object": "upgrade-proxysql-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2026-04-08T18:57:54.100Z	INFO	statefulSet was changed, run smart update	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a"}
2026-04-08T18:57:54.105Z	INFO	primary pod	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-0.upgrade-proxysql-pxc.upgrade-proxysql-4143.svc.cluster.local"}
2026-04-08T18:57:54.106Z	INFO	apply changes to secondary pod	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-2"}
2026-04-08T18:57:54.561Z	ERROR	sync users	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "2b61ec44-fac4-47c0-945c-1bb7042132ff", "error": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") /  / ", "errorVerbose": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") /  / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:975\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:832\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2026-04-08T18:57:54.779Z	INFO	Pod is not updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-2", "currentRevision": "upgrade-proxysql-pxc-7bb49f8b67", "targetRevision": "upgrade-proxysql-pxc-6576b69996"}
2026-04-08T18:58:04.876Z	INFO	Pod is not updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-2", "currentRevision": "upgrade-proxysql-pxc-7bb49f8b67", "targetRevision": "upgrade-proxysql-pxc-6576b69996"}
2026-04-08T18:58:14.879Z	INFO	Pod is not running	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-2", "phase": "Pending"}
2026-04-08T18:58:14.879Z	INFO	pod is waiting	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-2", "reason": "PodInitializing"}
2026-04-08T18:58:54.880Z	INFO	Pod is updated, running and ready	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-2"}
2026-04-08T18:58:54.895Z	INFO	waiting for pod to be present in hostgroup	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-2", "host": "upgrade-proxysql-pxc-2.upgrade-proxysql-pxc.upgrade-proxysql-4143", "hostgroup": "10"}
2026-04-08T18:59:04.897Z	INFO	pod present in hostgroup	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-2", "host": "upgrade-proxysql-pxc-2.upgrade-proxysql-pxc.upgrade-proxysql-4143", "hostgroup": "10"}
2026-04-08T18:59:04.900Z	INFO	pod is online	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-2"}
2026-04-08T18:59:04.901Z	INFO	apply changes to secondary pod	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-1"}
2026-04-08T18:59:05.076Z	INFO	Pod is not updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-1", "currentRevision": "upgrade-proxysql-pxc-7bb49f8b67", "targetRevision": "upgrade-proxysql-pxc-6576b69996"}
2026-04-08T18:59:15.169Z	INFO	Pod is not updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-1", "currentRevision": "upgrade-proxysql-pxc-7bb49f8b67", "targetRevision": "upgrade-proxysql-pxc-6576b69996"}
2026-04-08T18:59:25.151Z	INFO	Pod is not running	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-1", "phase": "Pending"}
2026-04-08T18:59:25.151Z	INFO	pod is waiting	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-1", "reason": "PodInitializing"}
2026-04-08T19:00:05.191Z	INFO	Pod is updated, running and ready	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-1"}
2026-04-08T19:00:05.207Z	INFO	pod present in hostgroup	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-1", "host": "upgrade-proxysql-pxc-1.upgrade-proxysql-pxc.upgrade-proxysql-4143", "hostgroup": "10"}
2026-04-08T19:00:05.210Z	INFO	apply changes to primary pod	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-0"}
2026-04-08T19:00:05.210Z	INFO	pod is online	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-1"}
2026-04-08T19:00:05.441Z	INFO	Pod is not updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-0", "currentRevision": "upgrade-proxysql-pxc-7bb49f8b67", "targetRevision": "upgrade-proxysql-pxc-6576b69996"}
2026-04-08T19:00:15.547Z	INFO	Pod is not updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-0", "currentRevision": "upgrade-proxysql-pxc-7bb49f8b67", "targetRevision": "upgrade-proxysql-pxc-6576b69996"}
2026-04-08T19:00:25.524Z	INFO	Pod is not running	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-0", "phase": "Pending"}
2026-04-08T19:00:25.524Z	INFO	pod is waiting	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-0", "reason": "PodInitializing"}
2026-04-08T19:00:35.515Z	INFO	Pod is not running	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-0", "phase": "Pending"}
2026-04-08T19:00:35.515Z	INFO	pod is waiting	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-0", "reason": "PodInitializing"}
2026-04-08T19:01:15.553Z	INFO	Pod is updated, running and ready	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-0"}
2026-04-08T19:01:15.572Z	INFO	pod present in hostgroup	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-0", "host": "upgrade-proxysql-pxc-0.upgrade-proxysql-pxc.upgrade-proxysql-4143", "hostgroup": "11"}
2026-04-08T19:01:15.575Z	INFO	pod is online	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "pod": "upgrade-proxysql-pxc-0"}
2026-04-08T19:01:15.575Z	INFO	smart update finished	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a"}
2026-04-08T19:01:15.622Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "8c49db06-86cb-4dd4-9957-e4a6ffb6ce7a", "new version": "5.7.44-48-57"}
2026-04-08T19:01:18.546Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "2514b29e-ca15-46b9-80ed-166345f53742"}
2026-04-08T19:01:24.572Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "b6f2ceb0-48b2-4794-9933-e40d44550612"}
2026-04-08T19:01:30.826Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "5ad98bb3-2131-4e5a-95b6-df9b501c6e9a"}
2026-04-08T19:01:36.814Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "fff9752f-6e64-42d7-819d-89701c9478d3"}
2026-04-08T19:01:43.248Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "d235812d-80c3-4e66-9001-4b10c14d5178"}
2026-04-08T19:01:49.709Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "9762e751-cef1-4453-aabc-1ff8bcb082c4"}
2026-04-08T19:01:56.036Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-4143"}, "namespace": "upgrade-proxysql-4143", "name": "upgrade-proxysql", "reconcileID": "a825eb91-b5c0-479a-b12b-df5aa349d3e1"}
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:834
+ grep -v NAMESPACE
+ kubectl get pxc --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n upgrade-proxysql-4143 upgrade-proxysql --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/upgrade-proxysql patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.lJdJ8bPPHw
++ mktemp
+ local LAST_ERR=/tmp/tmp.kK4ROTGTTj
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lJdJ8bPPHw
perconaxtradbcluster.pxc.percona.com "upgrade-proxysql" deleted from upgrade-proxysql-4143 namespace
+ cat /tmp/tmp.kK4ROTGTTj
+ rm /tmp/tmp.lJdJ8bPPHw /tmp/tmp.kK4ROTGTTj
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.GdisAYngC2
++ mktemp
+ local LAST_ERR=/tmp/tmp.KkuIzmqFDn
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.GdisAYngC2
No resources found
+ cat /tmp/tmp.KkuIzmqFDn
+ rm /tmp/tmp.GdisAYngC2 /tmp/tmp.KkuIzmqFDn
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.ktpDE7Gwmq
++ mktemp
+ local LAST_ERR=/tmp/tmp.56Lfixo4m6
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ktpDE7Gwmq
No resources found
+ cat /tmp/tmp.56Lfixo4m6
+ rm /tmp/tmp.ktpDE7Gwmq /tmp/tmp.56Lfixo4m6
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.HPtp7F1SVC
++ mktemp
+ local LAST_ERR=/tmp/tmp.g2VXg4S2yY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.HPtp7F1SVC
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.g2VXg4S2yY
+ rm /tmp/tmp.HPtp7F1SVC /tmp/tmp.g2VXg4S2yY
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace upgrade-proxysql-4143
+ rm -rf /tmp/tmp.9XEC036CwX
++ mktemp
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.tKKzRABsXY
++ mktemp
+ local LAST_OUT=/tmp/tmp.HxfOgG9tBx
+ local LAST_ERR=/tmp/tmp.t5pqXPNJU4
+ local exit_status=0
++ kubectl get crd
++ mktemp
++ grep perconaxtradbcluster
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.NdBFwd8B6R
+ local exit_status=0
++ awk '{print $1}'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace upgrade-proxysql-4143
++ tr '\n' ' '
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator
+ kubectl delete crd perconaxtradbclusterbackups.pxc.percona.com perconaxtradbclusterrestores.pxc.percona.com perconaxtradbclusters.pxc.percona.com
customresourcedefinition.apiextensions.k8s.io "perconaxtradbclusterbackups.pxc.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaxtradbclusterrestores.pxc.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaxtradbclusters.pxc.percona.com" deleted
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------