Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/logs/upgrade-proxysql-5-7.log
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
+ CLUSTER=upgrade-proxysql
+ CLUSTER_SIZE=3
+ TARGET_OPERATOR_VER=1.19.0
+ TARGET_IMAGE=perconalab/percona-xtradb-cluster-operator:PR-2316-524c74d8
+ TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc5.7
+ TARGET_IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest
+ TARGET_IMAGE_PROXY=perconalab/percona-xtradb-cluster-operator:main-proxysql
+ TARGET_IMAGE_HAPROXY=perconalab/percona-xtradb-cluster-operator:main-haproxy
+ TARGET_IMAGE_BACKUP=perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup
+ TARGET_IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]]
++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/'
++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc5.7
+ PXC_VER=5.7
++ curl -s https://check.percona.com/versions/v1/pxc-operator
++ tail -n1
++ jq -r '.versions[].operator'
++ sort -V
+ INIT_OPERATOR_VER=1.18.0
+ [[ 1.18.0 == \1\.\1\9\.\0 ]]
+ GIT_TAG=v1.18.0
++ curl -s 'https://check.percona.com/versions/v1/pxc-operator/1.18.0/latest?databaseVersion=5.7'
+ INIT_OPERATOR_IMAGES='{"versions":[{"product":"pxc-operator","operator":"1.18.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"","status":"recommended","critical":false},"3.3.1":{"imagePath":"percona/pmm-client:3.3.1","imageHash":"29a9bb1c69fef8bedc4d4a9ed0ae8224a8623fd3eb8676ef40b13fd044188cb4","imageHashArm64":"","status":"recommended","critical":false}},"proxysql":{"2.7.3":{"imagePath":"percona/proxysql2:2.7.3","imageHash":"51fedf9de05e4f130d5b08388511536fb1e1050a24ffc21bedb0f0b61a236567","imageHashArm64":"","status":"recommended","critical":false}},"haproxy":{"2.8.15":{"imagePath":"percona/haproxy:2.8.15","imageHash":"49e6987a1c8b27e9111ae1f1168dd51f2840eb6d939ffc157358f0f259819006","imageHashArm64":"","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.18.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.18.0","imageHash":"0eca0b096482c7d09792c15fee00dbdcd0fbf3cd487dab60eb2774b025681e85","imageHashArm64":"bdb7a0ff6b78e98b16f8b521e91682202b6d404202283b34b8168013d5c06356","status":"recommended","critical":false}},"logCollector":{"4.0.1":{"imagePath":"percona/fluentbit:4.0.1","imageHash":"a4ab7dd10379ccf74607f6b05225c4996eeff53b628bda94e615781a1f58b779","imageHashArm64":"","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
+ OPERATOR_NAME=percona-xtradb-cluster-operator
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.18.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"","status":"recommended","critical":false},"3.3.1":{"imagePath":"percona/pmm-client:3.3.1","imageHash":"29a9bb1c69fef8bedc4d4a9ed0ae8224a8623fd3eb8676ef40b13fd044188cb4","imageHashArm64":"","status":"recommended","critical":false}},"proxysql":{"2.7.3":{"imagePath":"percona/proxysql2:2.7.3","imageHash":"51fedf9de05e4f130d5b08388511536fb1e1050a24ffc21bedb0f0b61a236567","imageHashArm64":"","status":"recommended","critical":false}},"haproxy":{"2.8.15":{"imagePath":"percona/haproxy:2.8.15","imageHash":"49e6987a1c8b27e9111ae1f1168dd51f2840eb6d939ffc157358f0f259819006","imageHashArm64":"","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.18.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.18.0","imageHash":"0eca0b096482c7d09792c15fee00dbdcd0fbf3cd487dab60eb2774b025681e85","imageHashArm64":"bdb7a0ff6b78e98b16f8b521e91682202b6d404202283b34b8168013d5c06356","status":"recommended","critical":false}},"logCollector":{"4.0.1":{"imagePath":"percona/fluentbit:4.0.1","imageHash":"a4ab7dd10379ccf74607f6b05225c4996eeff53b628bda94e615781a1f58b779","imageHashArm64":"","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
++ jq -r '.versions[].matrix.operator[].imagePath'
+ IMAGE=percona/percona-xtradb-cluster-operator:1.18.0
++ echo perconalab/percona-xtradb-cluster-operator:PR-2316-524c74d8
++ cut -d/ -f1
+ [[ perconalab == \p\e\r\c\o\n\a\l\a\b ]]
+ IMAGE=perconalab/percona-xtradb-cluster-operator:1.18.0
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.18.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"","status":"recommended","critical":false},"3.3.1":{"imagePath":"percona/pmm-client:3.3.1","imageHash":"29a9bb1c69fef8bedc4d4a9ed0ae8224a8623fd3eb8676ef40b13fd044188cb4","imageHashArm64":"","status":"recommended","critical":false}},"proxysql":{"2.7.3":{"imagePath":"percona/proxysql2:2.7.3","imageHash":"51fedf9de05e4f130d5b08388511536fb1e1050a24ffc21bedb0f0b61a236567","imageHashArm64":"","status":"recommended","critical":false}},"haproxy":{"2.8.15":{"imagePath":"percona/haproxy:2.8.15","imageHash":"49e6987a1c8b27e9111ae1f1168dd51f2840eb6d939ffc157358f0f259819006","imageHashArm64":"","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.18.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.18.0","imageHash":"0eca0b096482c7d09792c15fee00dbdcd0fbf3cd487dab60eb2774b025681e85","imageHashArm64":"bdb7a0ff6b78e98b16f8b521e91682202b6d404202283b34b8168013d5c06356","status":"recommended","critical":false}},"logCollector":{"4.0.1":{"imagePath":"percona/fluentbit:4.0.1","imageHash":"a4ab7dd10379ccf74607f6b05225c4996eeff53b628bda94e615781a1f58b779","imageHashArm64":"","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
++ jq -r '.versions[].matrix.pxc[].imagePath'
+ IMAGE_PXC=percona/percona-xtradb-cluster:5.7.44-31.65
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.18.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"","status":"recommended","critical":false},"3.3.1":{"imagePath":"percona/pmm-client:3.3.1","imageHash":"29a9bb1c69fef8bedc4d4a9ed0ae8224a8623fd3eb8676ef40b13fd044188cb4","imageHashArm64":"","status":"recommended","critical":false}},"proxysql":{"2.7.3":{"imagePath":"percona/proxysql2:2.7.3","imageHash":"51fedf9de05e4f130d5b08388511536fb1e1050a24ffc21bedb0f0b61a236567","imageHashArm64":"","status":"recommended","critical":false}},"haproxy":{"2.8.15":{"imagePath":"percona/haproxy:2.8.15","imageHash":"49e6987a1c8b27e9111ae1f1168dd51f2840eb6d939ffc157358f0f259819006","imageHashArm64":"","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.18.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.18.0","imageHash":"0eca0b096482c7d09792c15fee00dbdcd0fbf3cd487dab60eb2774b025681e85","imageHashArm64":"bdb7a0ff6b78e98b16f8b521e91682202b6d404202283b34b8168013d5c06356","status":"recommended","critical":false}},"logCollector":{"4.0.1":{"imagePath":"percona/fluentbit:4.0.1","imageHash":"a4ab7dd10379ccf74607f6b05225c4996eeff53b628bda94e615781a1f58b779","imageHashArm64":"","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
++ jq -r '.versions[].matrix.pmm[].imagePath'
++ tail -n1
+ IMAGE_PMM_CLIENT=percona/pmm-client:3.3.1
++ jq -r '.versions[].matrix.proxysql[].imagePath'
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.18.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"","status":"recommended","critical":false},"3.3.1":{"imagePath":"percona/pmm-client:3.3.1","imageHash":"29a9bb1c69fef8bedc4d4a9ed0ae8224a8623fd3eb8676ef40b13fd044188cb4","imageHashArm64":"","status":"recommended","critical":false}},"proxysql":{"2.7.3":{"imagePath":"percona/proxysql2:2.7.3","imageHash":"51fedf9de05e4f130d5b08388511536fb1e1050a24ffc21bedb0f0b61a236567","imageHashArm64":"","status":"recommended","critical":false}},"haproxy":{"2.8.15":{"imagePath":"percona/haproxy:2.8.15","imageHash":"49e6987a1c8b27e9111ae1f1168dd51f2840eb6d939ffc157358f0f259819006","imageHashArm64":"","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.18.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.18.0","imageHash":"0eca0b096482c7d09792c15fee00dbdcd0fbf3cd487dab60eb2774b025681e85","imageHashArm64":"bdb7a0ff6b78e98b16f8b521e91682202b6d404202283b34b8168013d5c06356","status":"recommended","critical":false}},"logCollector":{"4.0.1":{"imagePath":"percona/fluentbit:4.0.1","imageHash":"a4ab7dd10379ccf74607f6b05225c4996eeff53b628bda94e615781a1f58b779","imageHashArm64":"","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
+ IMAGE_PROXY=percona/proxysql2:2.7.3
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.18.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"","status":"recommended","critical":false},"3.3.1":{"imagePath":"percona/pmm-client:3.3.1","imageHash":"29a9bb1c69fef8bedc4d4a9ed0ae8224a8623fd3eb8676ef40b13fd044188cb4","imageHashArm64":"","status":"recommended","critical":false}},"proxysql":{"2.7.3":{"imagePath":"percona/proxysql2:2.7.3","imageHash":"51fedf9de05e4f130d5b08388511536fb1e1050a24ffc21bedb0f0b61a236567","imageHashArm64":"","status":"recommended","critical":false}},"haproxy":{"2.8.15":{"imagePath":"percona/haproxy:2.8.15","imageHash":"49e6987a1c8b27e9111ae1f1168dd51f2840eb6d939ffc157358f0f259819006","imageHashArm64":"","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.18.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.18.0","imageHash":"0eca0b096482c7d09792c15fee00dbdcd0fbf3cd487dab60eb2774b025681e85","imageHashArm64":"bdb7a0ff6b78e98b16f8b521e91682202b6d404202283b34b8168013d5c06356","status":"recommended","critical":false}},"logCollector":{"4.0.1":{"imagePath":"percona/fluentbit:4.0.1","imageHash":"a4ab7dd10379ccf74607f6b05225c4996eeff53b628bda94e615781a1f58b779","imageHashArm64":"","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
++ jq -r '.versions[].matrix.haproxy[].imagePath'
+ IMAGE_HAPROXY=percona/haproxy:2.8.15
++ jq -r '.versions[].matrix.backup[].imagePath'
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.18.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"","status":"recommended","critical":false},"3.3.1":{"imagePath":"percona/pmm-client:3.3.1","imageHash":"29a9bb1c69fef8bedc4d4a9ed0ae8224a8623fd3eb8676ef40b13fd044188cb4","imageHashArm64":"","status":"recommended","critical":false}},"proxysql":{"2.7.3":{"imagePath":"percona/proxysql2:2.7.3","imageHash":"51fedf9de05e4f130d5b08388511536fb1e1050a24ffc21bedb0f0b61a236567","imageHashArm64":"","status":"recommended","critical":false}},"haproxy":{"2.8.15":{"imagePath":"percona/haproxy:2.8.15","imageHash":"49e6987a1c8b27e9111ae1f1168dd51f2840eb6d939ffc157358f0f259819006","imageHashArm64":"","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.18.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.18.0","imageHash":"0eca0b096482c7d09792c15fee00dbdcd0fbf3cd487dab60eb2774b025681e85","imageHashArm64":"bdb7a0ff6b78e98b16f8b521e91682202b6d404202283b34b8168013d5c06356","status":"recommended","critical":false}},"logCollector":{"4.0.1":{"imagePath":"percona/fluentbit:4.0.1","imageHash":"a4ab7dd10379ccf74607f6b05225c4996eeff53b628bda94e615781a1f58b779","imageHashArm64":"","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
+ IMAGE_BACKUP=percona/percona-xtrabackup:2.4.29
++ jq -r '.versions[].matrix.logCollector[].imagePath'
++ echo '{"versions":[{"product":"pxc-operator","operator":"1.18.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"","status":"recommended","critical":false},"3.3.1":{"imagePath":"percona/pmm-client:3.3.1","imageHash":"29a9bb1c69fef8bedc4d4a9ed0ae8224a8623fd3eb8676ef40b13fd044188cb4","imageHashArm64":"","status":"recommended","critical":false}},"proxysql":{"2.7.3":{"imagePath":"percona/proxysql2:2.7.3","imageHash":"51fedf9de05e4f130d5b08388511536fb1e1050a24ffc21bedb0f0b61a236567","imageHashArm64":"","status":"recommended","critical":false}},"haproxy":{"2.8.15":{"imagePath":"percona/haproxy:2.8.15","imageHash":"49e6987a1c8b27e9111ae1f1168dd51f2840eb6d939ffc157358f0f259819006","imageHashArm64":"","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.18.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.18.0","imageHash":"0eca0b096482c7d09792c15fee00dbdcd0fbf3cd487dab60eb2774b025681e85","imageHashArm64":"bdb7a0ff6b78e98b16f8b521e91682202b6d404202283b34b8168013d5c06356","status":"recommended","critical":false}},"logCollector":{"4.0.1":{"imagePath":"percona/fluentbit:4.0.1","imageHash":"a4ab7dd10379ccf74607f6b05225c4996eeff53b628bda94e615781a1f58b779","imageHashArm64":"","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}'
+ IMAGE_LOGCOLLECTOR=percona/fluentbit:4.0.1
+ [[ -n '' ]]
+ [[ 1.19.0 == \1\.\1\8\.\0 ]]
+ main
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.40p2wHbo5U
++ mktemp
+ local LAST_ERR=/tmp/tmp.y4mBJSi1jM
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.40p2wHbo5U
namespace/cert-manager created
+ cat /tmp/tmp.y4mBJSi1jM
+ rm /tmp/tmp.40p2wHbo5U /tmp/tmp.y4mBJSi1jM
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.DAvTf0aciQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.uXkRlFYqqc
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.DAvTf0aciQ
namespace/cert-manager labeled
+ cat /tmp/tmp.uXkRlFYqqc
+ rm /tmp/tmp.DAvTf0aciQ /tmp/tmp.uXkRlFYqqc
+ return 0
+ kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.18.2/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.e7C06WTQn3
++ mktemp
+ local LAST_ERR=/tmp/tmp.rAcNeufKYY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.18.2/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.e7C06WTQn3
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-tokenrequest created
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager-cert-manager-tokenrequest created
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager-cainjector created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.rAcNeufKYY
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.e7C06WTQn3 /tmp/tmp.rAcNeufKYY
+ return 0
+ '[' '' == 4.10 ']'
+ sleep 70
+ create_infra_gh upgrade-proxysql-2190 v1.18.0
+ local ns=upgrade-proxysql-2190
+ local git_tag=v1.18.0
+ '[' -n pxc-operator ']'
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl api-resources
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.2bszLVljNd
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.kgYAENNE5w
+ local exit_status=0
++ seq 0 2
+ local LAST_OUT=/tmp/tmp.4QmPO39g0k
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.tT7JVBZvPE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ xargs kubectl delete ns
+ awk '{print$1}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2bszLVljNd
+ cat /tmp/tmp.kgYAENNE5w
+ rm /tmp/tmp.2bszLVljNd /tmp/tmp.kgYAENNE5w
+ return 0
namespace "cert-manager" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.4QmPO39g0k
namespace "pxc-operator" deleted
+ cat /tmp/tmp.tT7JVBZvPE
+ rm /tmp/tmp.4QmPO39g0k /tmp/tmp.tT7JVBZvPE
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.hY9KJZGeXy
++ mktemp
+ local LAST_ERR=/tmp/tmp.tstNwDu2v3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.hY9KJZGeXy
namespace/pxc-operator created
+ cat /tmp/tmp.tstNwDu2v3
+ rm /tmp/tmp.hY9KJZGeXy /tmp/tmp.tstNwDu2v3
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZuNkdO1SAT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LriKAMbOOd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZuNkdO1SAT
++ cat /tmp/tmp.LriKAMbOOd
++ rm /tmp/tmp.ZuNkdO1SAT /tmp/tmp.LriKAMbOOd
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2316-524c74d8-4-cluster6 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ostPI9EWOk
++ mktemp
+ local LAST_ERR=/tmp/tmp.uEysfvYYax
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2316-524c74d8-4-cluster6 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ostPI9EWOk
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2316-524c74d8-4-cluster6" modified.
+ cat /tmp/tmp.uEysfvYYax
+ rm /tmp/tmp.ostPI9EWOk /tmp/tmp.uEysfvYYax
+ return 0
+ deploy_operator_gh v1.18.0
+ local git_tag=v1.18.0
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

++ kubectl_bin get crds -o 'jsonpath={.items[?(@.metadata.name == "perconaxtradbclusters.pxc.percona.com")].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AF84ozTD2S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UzaJWbcLM3
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crds -o 'jsonpath={.items[?(@.metadata.name == "perconaxtradbclusters.pxc.percona.com")].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AF84ozTD2S
++ cat /tmp/tmp.UzaJWbcLM3
++ rm /tmp/tmp.AF84ozTD2S /tmp/tmp.UzaJWbcLM3
++ return 0
+ [[ -n perconaxtradbclusters.pxc.percona.com ]]
++ kubectl_bin get crd/perconaxtradbclusters.pxc.percona.com -o 'jsonpath={.spec.versions[?(@.name == "v1-18-0")].name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4RkXlEVhAr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k8bbwPu5gw
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/perconaxtradbclusters.pxc.percona.com -o 'jsonpath={.spec.versions[?(@.name == "v1-18-0")].name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4RkXlEVhAr
++ cat /tmp/tmp.k8bbwPu5gw
++ rm /tmp/tmp.4RkXlEVhAr /tmp/tmp.k8bbwPu5gw
++ return 0
+ [[ -n '' ]]
+ kubectl_bin apply --server-side --force-conflicts -f https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.18.0/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.DrBn3TOYzd
++ mktemp
+ local LAST_ERR=/tmp/tmp.47c7G71p56
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.18.0/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.DrBn3TOYzd
+ cat /tmp/tmp.47c7G71p56
+ rm /tmp/tmp.DrBn3TOYzd /tmp/tmp.47c7G71p56
+ return 0
+ local rbac_yaml=rbac
+ local operator_yaml=operator.yaml
+ '[' -n pxc-operator ']'
+ rbac_yaml=cw-rbac
+ operator_yaml=cw-operator.yaml
+ apply_rbac_gh cw-rbac v1.18.0
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ local git_tag=v1.18.0
+ curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.18.0/deploy/cw-rbac.yaml
+ /usr/bin/sed -i -e 's^namespace: .*^namespace: pxc-operator^' /tmp/tmp.B2yRMevosb/rbac_v1.18.0.yaml
+ kubectl_bin apply -f /tmp/tmp.B2yRMevosb/rbac_v1.18.0.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.FggIETtbeZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.lsoucOMduc
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /tmp/tmp.B2yRMevosb/rbac_v1.18.0.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.FggIETtbeZ
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.lsoucOMduc
+ rm /tmp/tmp.FggIETtbeZ /tmp/tmp.lsoucOMduc
+ return 0
+ curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.18.0/deploy/cw-operator.yaml
+ cat /tmp/tmp.B2yRMevosb/cw-operator.yaml_v1.18.0.yaml
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:1.18.0^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"'
+ kubectl_bin apply -n pxc-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.UmPkdTUIk2
++ mktemp
+ local LAST_ERR=/tmp/tmp.kgR06wdZQk
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n pxc-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UmPkdTUIk2
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.kgR06wdZQk
+ rm /tmp/tmp.UmPkdTUIk2 /tmp/tmp.kgR06wdZQk
+ return 0
+ sleep 2
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ head -1
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6vypd99TeT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.l9NqSpl3iJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6vypd99TeT
++ cat /tmp/tmp.l9NqSpl3iJ
++ rm /tmp/tmp.6vypd99TeT /tmp/tmp.l9NqSpl3iJ
++ return 0
+ wait_pod percona-xtradb-cluster-operator-85f65db574-xtnx9
+ local pod=percona-xtradb-cluster-operator-85f65db574-xtnx9
+ local max_retry=480
+ local ns=
++ echo percona-xtradb-cluster-operator-85f65db574-xtnx9
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-85f65db574-xtnx9 condition met
waiting for pod/percona-xtradb-cluster-operator-85f65db574-xtnx9 to become Ready.Ok
+ create_namespace upgrade-proxysql-2190
+ local namespace=upgrade-proxysql-2190
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl api-resources
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces upgrade-proxysql-2190'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces upgrade-proxysql-2190
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace upgrade-proxysql-2190
+ awk '{print$1}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.HceBYx2xiB
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.d8M3GEyXEx
+ local exit_status=0
+ local LAST_OUT=/tmp/tmp.P4wl7LmKXu
++ seq 0 2
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-proxysql-2190
+ local LAST_ERR=/tmp/tmp.PMRo5zu8fw
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ xargs kubectl delete ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.P4wl7LmKXu
+ cat /tmp/tmp.PMRo5zu8fw
+ rm /tmp/tmp.P4wl7LmKXu /tmp/tmp.PMRo5zu8fw
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-proxysql-2190
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-proxysql-2190
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.HceBYx2xiB
+ cat /tmp/tmp.d8M3GEyXEx
Error from server (NotFound): namespaces "upgrade-proxysql-2190" not found
+ rm /tmp/tmp.HceBYx2xiB /tmp/tmp.d8M3GEyXEx
+ return 1
+ :
+ wait_for_delete namespace/upgrade-proxysql-2190
+ local res=namespace/upgrade-proxysql-2190
+ echo -n 'waiting for namespace/upgrade-proxysql-2190 to be deleted'
waiting for namespace/upgrade-proxysql-2190 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "upgrade-proxysql-2190" not found
+ desc 'create namespace upgrade-proxysql-2190'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace upgrade-proxysql-2190
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace upgrade-proxysql-2190
++ mktemp
+ local LAST_OUT=/tmp/tmp.5tFfixKm3v
++ mktemp
+ local LAST_ERR=/tmp/tmp.OWJgMtjW4R
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace upgrade-proxysql-2190
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5tFfixKm3v
namespace/upgrade-proxysql-2190 created
+ cat /tmp/tmp.OWJgMtjW4R
+ rm /tmp/tmp.5tFfixKm3v /tmp/tmp.OWJgMtjW4R
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mbeIlEjF4F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nmtfe4WnrE
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.mbeIlEjF4F
++ cat /tmp/tmp.nmtfe4WnrE
++ rm /tmp/tmp.mbeIlEjF4F /tmp/tmp.nmtfe4WnrE
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2316-524c74d8-4-cluster6 --namespace=upgrade-proxysql-2190
++ mktemp
+ local LAST_OUT=/tmp/tmp.uuLy1LprAA
++ mktemp
+ local LAST_ERR=/tmp/tmp.9mqIAMV1C0
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2316-524c74d8-4-cluster6 --namespace=upgrade-proxysql-2190
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.uuLy1LprAA
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2316-524c74d8-4-cluster6" modified.
+ cat /tmp/tmp.9mqIAMV1C0
+ rm /tmp/tmp.uuLy1LprAA /tmp/tmp.9mqIAMV1C0
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.GZAKVTLTZY
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZAMxKVRE3R
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.GZAKVTLTZY
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.ZAMxKVRE3R
+ rm /tmp/tmp.GZAKVTLTZY /tmp/tmp.ZAMxKVRE3R
+ return 0
+ local proxy=proxysql
+ local cr_yaml=/tmp/tmp.B2yRMevosb/cr_1.18.0_proxysql.yaml
+ prepare_cr_yaml /tmp/tmp.B2yRMevosb/cr_1.18.0_proxysql.yaml proxysql upgrade-proxysql 3 v1.18.0
+ local cr_yaml=/tmp/tmp.B2yRMevosb/cr_1.18.0_proxysql.yaml
+ local proxy=proxysql
+ local cluster=upgrade-proxysql
+ local cluster_size=3
+ local git_tag=v1.18.0
+ yq eval '
			.metadata.name = "upgrade-proxysql" |
			.spec.secretsName = "my-cluster-secrets" |
			.spec.vaultSecretName = "some-name-vault" |
			.spec.sslSecretName = "some-name-ssl" |
			.spec.sslInternalSecretName = "some-name-ssl-internal" |
			.spec.upgradeOptions.apply = "disabled" |
			.spec.pxc.size = 3 |
			.spec.proxysql.size = 3 |
			.spec.haproxy.size = 3 |
			.spec.pxc.image = "-pxc" |
			.spec.proxysql.image = "-proxysql" |
			.spec.haproxy.image = "-haproxy" |
			.spec.backup.image = "-backup" |
			.spec.backup.storages.minio.s3.credentialsSecret = "minio-secret" |
			.spec.backup.storages.minio.s3.region = "us-east-1" |
			.spec.backup.storages.minio.s3.bucket = "operator-testing" |
			.spec.backup.storages.minio.s3.endpointUrl = "http://minio-service.#namespace:9000/" |
			.spec.backup.storages.minio.type = "s3" |
			.spec.pmm.image = "-pmm"
		' -
+ curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.18.0/deploy/cr.yaml
+ [[ proxysql == \h\a\p\r\o\x\y ]]
+ yq -i eval '
			.spec.haproxy.enabled = false |
			.spec.proxysql.enabled = true
		' /tmp/tmp.B2yRMevosb/cr_1.18.0_proxysql.yaml
+ [[ -n '' ]]
+ spinup_pxc upgrade-proxysql /tmp/tmp.B2yRMevosb/cr_1.18.0_proxysql.yaml 3 30 /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/secrets_without_tls.yml
+ local cluster=upgrade-proxysql
+ local config=/tmp/tmp.B2yRMevosb/cr_1.18.0_proxysql.yaml
+ local size=3
+ local sleep=30
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/secrets_without_tls.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/secrets_without_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.tLp33MkWgx
++ mktemp
+ local LAST_ERR=/tmp/tmp.SSfN7HzAcs
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/secrets_without_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.tLp33MkWgx
secret/my-cluster-secrets created
+ cat /tmp/tmp.SSfN7HzAcs
+ rm /tmp/tmp.tLp33MkWgx /tmp/tmp.SSfN7HzAcs
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/client.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/client.yml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/client.yml ''
+ kubectl_bin apply -f -
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/client.yml
+ local pvc_name=
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:5.7.44-31.65#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:1.18.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: percona/pmm-client:3.3.1#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: percona/proxysql2:2.7.3#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: percona/haproxy:2.8.15#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:5.7.44-31.65#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: percona/fluentbit:4.0.1#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: percona/percona-xtrabackup:2.4.29#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
++ mktemp
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.upgrade-proxysql-2190~
+ local LAST_OUT=/tmp/tmp.TGHeGRRrVj
++ mktemp
+ local LAST_ERR=/tmp/tmp.kUGfRzhl9U
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.TGHeGRRrVj
deployment.apps/pxc-client created
+ cat /tmp/tmp.kUGfRzhl9U
+ rm /tmp/tmp.TGHeGRRrVj /tmp/tmp.kUGfRzhl9U
+ return 0
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ upgrade-proxysql == \d\e\m\a\n\d\-\b\a\c\k\u\p ]]
+ [[ upgrade-proxysql == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]]
+ apply_config /tmp/tmp.B2yRMevosb/cr_1.18.0_proxysql.yaml
+ local config_file=/tmp/tmp.B2yRMevosb/cr_1.18.0_proxysql.yaml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /tmp/tmp.B2yRMevosb/cr_1.18.0_proxysql.yaml ''
+ local input_file=/tmp/tmp.B2yRMevosb/cr_1.18.0_proxysql.yaml
+ local pvc_name=
+ kubectl_bin apply -f -
+ cat /tmp/tmp.B2yRMevosb/cr_1.18.0_proxysql.yaml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:1.18.0#'
+ local LAST_OUT=/tmp/tmp.jwlgtoGi8l
+ /usr/bin/sed -e 's#image:.*-pmm$#image: percona/pmm-client:3.3.1#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: percona/percona-xtrabackup:2.4.29#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: percona/proxysql2:2.7.3#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: percona/haproxy:2.8.15#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:5.7.44-31.65#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: percona/fluentbit:4.0.1#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.upgrade-proxysql-2190~
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZNWfy9cE2W
+ local exit_status=0
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
++ seq 0 2
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:5.7.44-31.65#'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.jwlgtoGi8l
perconaxtradbcluster.pxc.percona.com/upgrade-proxysql created
+ cat /tmp/tmp.ZNWfy9cE2W
+ rm /tmp/tmp.jwlgtoGi8l /tmp/tmp.ZNWfy9cE2W
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy upgrade-proxysql
++ local target_cluster=upgrade-proxysql
+++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.J6gSiqGgGW
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.aLI8vB4rIJ
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.J6gSiqGgGW
+++ cat /tmp/tmp.aLI8vB4rIJ
+++ rm /tmp/tmp.J6gSiqGgGW /tmp/tmp.aLI8vB4rIJ
+++ return 0
++ [[ false == \t\r\u\e ]]
+++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.VGNNYQhqWV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.htriByVuyX
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.VGNNYQhqWV
+++ cat /tmp/tmp.htriByVuyX
+++ rm /tmp/tmp.VGNNYQhqWV /tmp/tmp.htriByVuyX
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo upgrade-proxysql-proxysql
++ return
+ local proxy=upgrade-proxysql-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n upgrade-proxysql-2190
++ mktemp
+ local LAST_OUT=/tmp/tmp.WrppAoQ5uD
++ mktemp
+ local LAST_ERR=/tmp/tmp.27qQBBOg13
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n upgrade-proxysql-2190
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n upgrade-proxysql-2190
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n upgrade-proxysql-2190
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.WrppAoQ5uD
+ cat /tmp/tmp.27qQBBOg13
error: no matching resources found
+ rm /tmp/tmp.WrppAoQ5uD /tmp/tmp.27qQBBOg13
+ return 1
+ true
+ wait_for_running upgrade-proxysql-proxysql 1
+ local name=upgrade-proxysql-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-proxysql-0 480
+ local pod=upgrade-proxysql-proxysql-0
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo upgrade-proxysql-proxysql-0
+ local container=proxysql
+ set +o xtrace
pod/upgrade-proxysql-proxysql-0 condition met
waiting for pod/upgrade-proxysql-proxysql-0 to become Ready.Ok
+ wait_for_running upgrade-proxysql-pxc 3
+ local name=upgrade-proxysql-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-0 480
+ local pod=upgrade-proxysql-pxc-0
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-0
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-0 condition met
waiting for pod/upgrade-proxysql-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-1 480
+ local pod=upgrade-proxysql-pxc-1
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-1
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-1 condition met
waiting for pod/upgrade-proxysql-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-2 480
+ local pod=upgrade-proxysql-pxc-2
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ echo upgrade-proxysql-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-2 condition met
waiting for pod/upgrade-proxysql-pxc-2 to become Ready.Ok
+ sleep 30
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VkINM1DppK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.T4vezu1KKs
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VkINM1DppK
++ cat /tmp/tmp.T4vezu1KKs
++ rm /tmp/tmp.VkINM1DppK /tmp/tmp.T4vezu1KKs
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
++ is_keyring_plugin_in_use upgrade-proxysql
++ local cluster=upgrade-proxysql
++ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
++ kubectl exec -it upgrade-proxysql-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ grep -E -o 'early-plugin-load=keyring_\w+.so'
Unable to use a TTY - input is not a terminal or the right kind of file
++ return 1
+ [[ -n '' ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h upgrade-proxysql-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h upgrade-proxysql-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YaJnTS1YJo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.U05S2Sj8zZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YaJnTS1YJo
++ cat /tmp/tmp.U05S2Sj8zZ
++ rm /tmp/tmp.YaJnTS1YJo /tmp/tmp.U05S2Sj8zZ
++ return 0
+ client_pod=pxc-client-cdffbf747-66td6
+ wait_pod pxc-client-cdffbf747-66td6
+ local pod=pxc-client-cdffbf747-66td6
+ local max_retry=480
+ local ns=
++ echo pxc-client-cdffbf747-66td6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-cdffbf747-66td6 condition met
waiting for pod/pxc-client-cdffbf747-66td6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h upgrade-proxysql-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h upgrade-proxysql-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VK6QE5UEaQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q1s6q5V2XP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VK6QE5UEaQ
++ cat /tmp/tmp.q1s6q5V2XP
++ rm /tmp/tmp.VK6QE5UEaQ /tmp/tmp.q1s6q5V2XP
++ return 0
+ client_pod=pxc-client-cdffbf747-66td6
+ wait_pod pxc-client-cdffbf747-66td6
+ local pod=pxc-client-cdffbf747-66td6
+ local max_retry=480
+ local ns=
++ echo pxc-client-cdffbf747-66td6
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-cdffbf747-66td6 condition met
waiting for pod/pxc-client-cdffbf747-66td6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9rjniv0DsM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IGqcfn5VCw
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.9rjniv0DsM
++ cat /tmp/tmp.IGqcfn5VCw
++ rm /tmp/tmp.9rjniv0DsM /tmp/tmp.IGqcfn5VCw
++ return 0
+ client_pod=pxc-client-cdffbf747-66td6
+ wait_pod pxc-client-cdffbf747-66td6
+ local pod=pxc-client-cdffbf747-66td6
+ local max_retry=480
+ local ns=
++ echo pxc-client-cdffbf747-66td6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-cdffbf747-66td6 condition met
waiting for pod/pxc-client-cdffbf747-66td6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.B2yRMevosb/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.B2yRMevosb/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.B2yRMevosb/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fILwo9sQaq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rPRk6HrB5q
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.fILwo9sQaq
++ cat /tmp/tmp.rPRk6HrB5q
++ rm /tmp/tmp.fILwo9sQaq /tmp/tmp.rPRk6HrB5q
++ return 0
+ client_pod=pxc-client-cdffbf747-66td6
+ wait_pod pxc-client-cdffbf747-66td6
+ local pod=pxc-client-cdffbf747-66td6
+ local max_retry=480
+ local ns=
++ echo pxc-client-cdffbf747-66td6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-cdffbf747-66td6 condition met
waiting for pod/pxc-client-cdffbf747-66td6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.B2yRMevosb/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.B2yRMevosb/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.B2yRMevosb/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zJVFakoyur
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LuRTSov3iD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zJVFakoyur
++ cat /tmp/tmp.LuRTSov3iD
++ rm /tmp/tmp.zJVFakoyur /tmp/tmp.LuRTSov3iD
++ return 0
+ client_pod=pxc-client-cdffbf747-66td6
+ wait_pod pxc-client-cdffbf747-66td6
+ local pod=pxc-client-cdffbf747-66td6
+ local max_retry=480
+ local ns=
++ echo pxc-client-cdffbf747-66td6
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-cdffbf747-66td6 condition met
waiting for pod/pxc-client-cdffbf747-66td6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.B2yRMevosb/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.B2yRMevosb/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.B2yRMevosb/select-1.sql
+ is_keyring_plugin_in_use upgrade-proxysql
+ local cluster=upgrade-proxysql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ kubectl exec -it upgrade-proxysql-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
+ grep -E -o 'early-plugin-load=keyring_\w+.so'
Unable to use a TTY - input is not a terminal or the right kind of file
+ return 1
+ compare_generation 1 proxysql upgrade-proxysql
+ local generation=1
+ local proxy=proxysql
+ local cluster=upgrade-proxysql
+ local current_generation
+ [[ proxysql == \h\a\p\r\o\x\y ]]
+ containers=(pxc proxysql)
+ for container in '"${containers[@]}"'
+ check_generation 1 pxc upgrade-proxysql
+ local generation=1
+ local container=pxc
+ local cluster=upgrade-proxysql
+ local current_generation
++ kubectl_bin get statefulset upgrade-proxysql-pxc -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MafC52KRKh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GMTBCm0bUv
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset upgrade-proxysql-pxc -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.MafC52KRKh
++ cat /tmp/tmp.GMTBCm0bUv
++ rm /tmp/tmp.MafC52KRKh /tmp/tmp.GMTBCm0bUv
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ for container in '"${containers[@]}"'
+ check_generation 1 proxysql upgrade-proxysql
+ local generation=1
+ local container=proxysql
+ local cluster=upgrade-proxysql
+ local current_generation
++ kubectl_bin get statefulset upgrade-proxysql-proxysql -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bGM9XqtF2j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NHa5tJzL83
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset upgrade-proxysql-proxysql -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bGM9XqtF2j
++ cat /tmp/tmp.NHa5tJzL83
++ rm /tmp/tmp.bGM9XqtF2j /tmp/tmp.NHa5tJzL83
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ desc 'upgrade operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
upgrade operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.yvTVExYvgK
++ mktemp
+ local LAST_ERR=/tmp/tmp.NGbipBVuxa
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.yvTVExYvgK
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.NGbipBVuxa
+ rm /tmp/tmp.yvTVExYvgK /tmp/tmp.NGbipBVuxa
+ return 0
+ [[ -n pxc-operator ]]
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.lO2WNSA54R
++ mktemp
+ local LAST_ERR=/tmp/tmp.NZcozh3eZJ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lO2WNSA54R
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.NZcozh3eZJ
+ rm /tmp/tmp.lO2WNSA54R /tmp/tmp.NZcozh3eZJ
+ return 0
+ kubectl_bin patch deployment percona-xtradb-cluster-operator '-p{"spec":{"template":{"spec":{"containers":[{"name":"percona-xtradb-cluster-operator","image":"perconalab/percona-xtradb-cluster-operator:PR-2316-524c74d8"}]}}}}' -n pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.h3BjLa4eGd
++ mktemp
+ local LAST_ERR=/tmp/tmp.lc2Yf0gLGp
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch deployment percona-xtradb-cluster-operator '-p{"spec":{"template":{"spec":{"containers":[{"name":"percona-xtradb-cluster-operator","image":"perconalab/percona-xtradb-cluster-operator:PR-2316-524c74d8"}]}}}}' -n pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.h3BjLa4eGd
deployment.apps/percona-xtradb-cluster-operator patched
+ cat /tmp/tmp.lc2Yf0gLGp
+ rm /tmp/tmp.h3BjLa4eGd /tmp/tmp.lc2Yf0gLGp
+ return 0
+ kubectl_bin rollout status deployment/percona-xtradb-cluster-operator -n pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZxXDt1Qmn6
++ mktemp
+ local LAST_ERR=/tmp/tmp.MPZOPV2sm3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl rollout status deployment/percona-xtradb-cluster-operator -n pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ZxXDt1Qmn6
Waiting for deployment "percona-xtradb-cluster-operator" rollout to finish: 0 of 1 updated replicas are available...
deployment "percona-xtradb-cluster-operator" successfully rolled out
+ cat /tmp/tmp.MPZOPV2sm3
+ rm /tmp/tmp.ZxXDt1Qmn6 /tmp/tmp.MPZOPV2sm3
+ return 0
+ sleep 10
+ desc 'wait for operator upgrade'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for operator upgrade
-----------------------------------------------------------------------------------

+ local i=0
+ local max=60
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image' -n pxc-operator
++ grep -vc NAME
++ awk '{print $1}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D14YUxPeuQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.c9zcflYtDU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.D14YUxPeuQ
++ cat /tmp/tmp.c9zcflYtDU
++ rm /tmp/tmp.D14YUxPeuQ /tmp/tmp.c9zcflYtDU
++ return 0
+ [[ 1 -eq 1 ]]
+ '[' -n pxc-operator ']'
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nu3N2LYEaz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VqtS88i1An
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nu3N2LYEaz
++ cat /tmp/tmp.VqtS88i1An
++ rm /tmp/tmp.nu3N2LYEaz /tmp/tmp.VqtS88i1An
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2316-524c74d8-4-cluster6 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.MtA8XfVDTr
++ mktemp
+ local LAST_ERR=/tmp/tmp.aGXjAtHuWR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2316-524c74d8-4-cluster6 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MtA8XfVDTr
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2316-524c74d8-4-cluster6" modified.
+ cat /tmp/tmp.aGXjAtHuWR
+ rm /tmp/tmp.MtA8XfVDTr /tmp/tmp.aGXjAtHuWR
+ return 0
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HgNsGdDeRe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PmcUbG8y6n
++ local exit_status=0
+++ seq 0 2
++ grep perconalab/percona-xtradb-cluster-operator:PR-2316-524c74d8
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image'
++ awk '{print $1}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.HgNsGdDeRe
++ cat /tmp/tmp.PmcUbG8y6n
++ rm /tmp/tmp.HgNsGdDeRe /tmp/tmp.PmcUbG8y6n
++ return 0
+ wait_pod percona-xtradb-cluster-operator-7d57c5dcc9-cnqc8
+ local pod=percona-xtradb-cluster-operator-7d57c5dcc9-cnqc8
+ local max_retry=480
+ local ns=
++ echo percona-xtradb-cluster-operator-7d57c5dcc9-cnqc8
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-7d57c5dcc9-cnqc8 condition met
waiting for pod/percona-xtradb-cluster-operator-7d57c5dcc9-cnqc8 to become Ready.Ok
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.czaOlUibgx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qddxDqbGZz
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.czaOlUibgx
++ cat /tmp/tmp.qddxDqbGZz
++ rm /tmp/tmp.czaOlUibgx /tmp/tmp.qddxDqbGZz
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2316-524c74d8-4-cluster6 --namespace=upgrade-proxysql-2190
++ mktemp
+ local LAST_OUT=/tmp/tmp.jfJS9j8BgH
++ mktemp
+ local LAST_ERR=/tmp/tmp.K2ehsMRqNh
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2316-524c74d8-4-cluster6 --namespace=upgrade-proxysql-2190
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.jfJS9j8BgH
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2316-524c74d8-4-cluster6" modified.
+ cat /tmp/tmp.K2ehsMRqNh
+ rm /tmp/tmp.jfJS9j8BgH /tmp/tmp.K2ehsMRqNh
+ return 0
+ desc 'check images and generation after operator upgrade'
+ set +o xtrace


-----------------------------------------------------------------------------------
check images and generation after operator upgrade
-----------------------------------------------------------------------------------

+ check_pxc_liveness upgrade-proxysql 3
+ local cluster=upgrade-proxysql
+ local cluster_size=3
+ wait_cluster_consistency upgrade-proxysql 3
+ local cluster_name=upgrade-proxysql
+ local cluster_size=3
+ local proxy_size=
+ '[' -z '' ']'
++ get_proxy_size upgrade-proxysql
++ local cluster=upgrade-proxysql
+++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.lM4REA9NuV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.E0ek7yODXq
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.lM4REA9NuV
+++ cat /tmp/tmp.E0ek7yODXq
+++ rm /tmp/tmp.lM4REA9NuV /tmp/tmp.E0ek7yODXq
+++ return 0
++ [[ false == \t\r\u\e ]]
+++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ZpDaSFU0P4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9yZ0xGb9tk
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.ZpDaSFU0P4
+++ cat /tmp/tmp.9yZ0xGb9tk
+++ rm /tmp/tmp.ZpDaSFU0P4 /tmp/tmp.9yZ0xGb9tk
+++ return 0
++ [[ true == \t\r\u\e ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.size}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mwOxv4HbCB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JLyd6GDZ0z
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.size}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.mwOxv4HbCB
++ cat /tmp/tmp.JLyd6GDZ0z
++ rm /tmp/tmp.mwOxv4HbCB /tmp/tmp.JLyd6GDZ0z
++ return 0
++ return
+ proxy_size=3
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=300
+ sleep 7
+ echo -n 'waiting for pxc/upgrade-proxysql to be ready'
waiting for pxc/upgrade-proxysql to be ready++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G0uwWvtKzm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3VEGccYiQ7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.G0uwWvtKzm
++ cat /tmp/tmp.3VEGccYiQ7
++ rm /tmp/tmp.G0uwWvtKzm /tmp/tmp.3VEGccYiQ7
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Tc1HcvM9ta
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iOvKTixVQl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Tc1HcvM9ta
++ cat /tmp/tmp.iOvKTixVQl
++ rm /tmp/tmp.Tc1HcvM9ta /tmp/tmp.iOvKTixVQl
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine upgrade-proxysql
+++ local cluster_name=upgrade-proxysql
++++ get_proxy upgrade-proxysql
++++ local target_cluster=upgrade-proxysql
+++++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.xJl5fLhQrk
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.3PQMfw56By
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.xJl5fLhQrk
+++++ cat /tmp/tmp.3PQMfw56By
+++++ rm /tmp/tmp.xJl5fLhQrk /tmp/tmp.3PQMfw56By
+++++ return 0
++++ [[ false == \t\r\u\e ]]
+++++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.Z7kJW2u7Mv
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.2aznZpMdTP
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.Z7kJW2u7Mv
+++++ cat /tmp/tmp.2aznZpMdTP
+++++ rm /tmp/tmp.Z7kJW2u7Mv /tmp/tmp.2aznZpMdTP
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo upgrade-proxysql-proxysql
++++ return
+++ local cluster_proxy=upgrade-proxysql-proxysql
+++ echo proxysql
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HRewFhOnlD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rleJA2qmyz
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.HRewFhOnlD
++ cat /tmp/tmp.rleJA2qmyz
++ rm /tmp/tmp.HRewFhOnlD /tmp/tmp.rleJA2qmyz
++ return 0
+ [[ 3 == \3 ]]
+ echo

+ wait_for_running upgrade-proxysql-pxc 3
+ local name=upgrade-proxysql-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-0 480
+ local pod=upgrade-proxysql-pxc-0
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-0 condition met
waiting for pod/upgrade-proxysql-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-1 480
+ local pod=upgrade-proxysql-pxc-1
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-1 condition met
waiting for pod/upgrade-proxysql-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-2 480
+ local pod=upgrade-proxysql-pxc-2
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-2 condition met
waiting for pod/upgrade-proxysql-pxc-2 to become Ready.Ok
++ seq 0 2
+ for i in '$(seq 0 $((cluster_size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fPUYSMaAQk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KdvMy15NYe
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.fPUYSMaAQk
++ cat /tmp/tmp.KdvMy15NYe
++ rm /tmp/tmp.fPUYSMaAQk /tmp/tmp.KdvMy15NYe
++ return 0
+ client_pod=pxc-client-cdffbf747-66td6
+ wait_pod pxc-client-cdffbf747-66td6
+ local pod=pxc-client-cdffbf747-66td6
+ local max_retry=480
+ local ns=
++ echo pxc-client-cdffbf747-66td6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-cdffbf747-66td6 condition met
waiting for pod/pxc-client-cdffbf747-66td6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.B2yRMevosb/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.B2yRMevosb/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.B2yRMevosb/select-1.sql
+ for i in '$(seq 0 $((cluster_size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JVTP3KYnqC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Pxu6T9kzFm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.JVTP3KYnqC
++ cat /tmp/tmp.Pxu6T9kzFm
++ rm /tmp/tmp.JVTP3KYnqC /tmp/tmp.Pxu6T9kzFm
++ return 0
+ client_pod=pxc-client-cdffbf747-66td6
+ wait_pod pxc-client-cdffbf747-66td6
+ local pod=pxc-client-cdffbf747-66td6
+ local max_retry=480
+ local ns=
++ echo pxc-client-cdffbf747-66td6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-cdffbf747-66td6 condition met
waiting for pod/pxc-client-cdffbf747-66td6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.B2yRMevosb/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.B2yRMevosb/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.B2yRMevosb/select-1.sql
+ for i in '$(seq 0 $((cluster_size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XCEdmqoElw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6i9vm4woAB
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XCEdmqoElw
++ cat /tmp/tmp.6i9vm4woAB
++ rm /tmp/tmp.XCEdmqoElw /tmp/tmp.6i9vm4woAB
++ return 0
+ client_pod=pxc-client-cdffbf747-66td6
+ wait_pod pxc-client-cdffbf747-66td6
+ local pod=pxc-client-cdffbf747-66td6
+ local max_retry=480
+ local ns=
++ echo pxc-client-cdffbf747-66td6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-cdffbf747-66td6 condition met
waiting for pod/pxc-client-cdffbf747-66td6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.B2yRMevosb/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.B2yRMevosb/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.B2yRMevosb/select-1.sql
++ kubectl_bin get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gVCSnJ9fIT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8xRKmuItZ2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gVCSnJ9fIT
++ cat /tmp/tmp.8xRKmuItZ2
++ rm /tmp/tmp.gVCSnJ9fIT /tmp/tmp.8xRKmuItZ2
++ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:PR-2316-524c74d8 == perconalab/percona-xtradb-cluster-operator:PR-2316-524c74d8 ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8RDFLPXpS0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1Qw9POsa94
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8RDFLPXpS0
++ cat /tmp/tmp.1Qw9POsa94
++ rm /tmp/tmp.8RDFLPXpS0 /tmp/tmp.1Qw9POsa94
++ return 0
+ [[ percona/proxysql2:2.7.3 == percona/proxysql2:2.7.3 ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N0VNzgjB67
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wVIbPiBKpp
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.N0VNzgjB67
++ cat /tmp/tmp.wVIbPiBKpp
++ rm /tmp/tmp.N0VNzgjB67 /tmp/tmp.wVIbPiBKpp
++ return 0
+ [[ percona/haproxy:2.8.15 == percona/haproxy:2.8.15 ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.backup.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.psNmxx9nL2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hEpzYINhAM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.backup.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.psNmxx9nL2
++ cat /tmp/tmp.hEpzYINhAM
++ rm /tmp/tmp.psNmxx9nL2 /tmp/tmp.hEpzYINhAM
++ return 0
+ [[ percona/percona-xtrabackup:2.4.29 == percona/percona-xtrabackup:2.4.29 ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.pmm.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nHDZUEZwMi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aHKt8wr2Ou
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.pmm.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nHDZUEZwMi
++ cat /tmp/tmp.aHKt8wr2Ou
++ rm /tmp/tmp.nHDZUEZwMi /tmp/tmp.aHKt8wr2Ou
++ return 0
+ [[ percona/pmm-client:3.3.1 == percona/pmm-client:3.3.1 ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.logcollector.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pr5ZIDNvvq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ew2RIjSZCX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.logcollector.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pr5ZIDNvvq
++ cat /tmp/tmp.ew2RIjSZCX
++ rm /tmp/tmp.pr5ZIDNvvq /tmp/tmp.ew2RIjSZCX
++ return 0
+ [[ percona/fluentbit:4.0.1 == percona/fluentbit:4.0.1 ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.pxc.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.awn8UfRfvp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8Kt3r0nxar
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.pxc.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.awn8UfRfvp
++ cat /tmp/tmp.8Kt3r0nxar
++ rm /tmp/tmp.awn8UfRfvp /tmp/tmp.8Kt3r0nxar
++ return 0
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 == percona/percona-xtradb-cluster:5.7.44-31.65 ]]
+ : Operator image has been updated correctly
+ compare_generation 1 proxysql upgrade-proxysql
+ local generation=1
+ local proxy=proxysql
+ local cluster=upgrade-proxysql
+ local current_generation
+ [[ proxysql == \h\a\p\r\o\x\y ]]
+ containers=(pxc proxysql)
+ for container in '"${containers[@]}"'
+ check_generation 1 pxc upgrade-proxysql
+ local generation=1
+ local container=pxc
+ local cluster=upgrade-proxysql
+ local current_generation
++ kubectl_bin get statefulset upgrade-proxysql-pxc -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EC5ziBdUfx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZndFIyMV3A
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset upgrade-proxysql-pxc -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.EC5ziBdUfx
++ cat /tmp/tmp.ZndFIyMV3A
++ rm /tmp/tmp.EC5ziBdUfx /tmp/tmp.ZndFIyMV3A
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ for container in '"${containers[@]}"'
+ check_generation 1 proxysql upgrade-proxysql
+ local generation=1
+ local container=proxysql
+ local cluster=upgrade-proxysql
+ local current_generation
++ kubectl_bin get statefulset upgrade-proxysql-proxysql -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dHtA0P28v4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d8eX6ISoLs
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset upgrade-proxysql-proxysql -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.dHtA0P28v4
++ cat /tmp/tmp.d8eX6ISoLs
++ rm /tmp/tmp.dHtA0P28v4 /tmp/tmp.d8eX6ISoLs
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ desc 'patch pxc images and upgrade'
+ set +o xtrace


-----------------------------------------------------------------------------------
patch pxc images and upgrade
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc upgrade-proxysql --type=merge --patch '{
        "spec": {
            "crVersion": "1.19.0",
            "pxc": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc5.7" },
            "pmm": { "image": "perconalab/pmm-client:dev-latest" },
            "haproxy": { "image": "perconalab/percona-xtradb-cluster-operator:main-haproxy" },
            "proxysql": { "image": "perconalab/percona-xtradb-cluster-operator:main-proxysql" },
            "logcollector": { "image": "perconalab/fluentbit:main-logcollector" },
            "backup": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup" }
        }}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.XcWaNVAI2A
++ mktemp
+ local LAST_ERR=/tmp/tmp.zSaWkpwHn4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc upgrade-proxysql --type=merge --patch '{
        "spec": {
            "crVersion": "1.19.0",
            "pxc": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc5.7" },
            "pmm": { "image": "perconalab/pmm-client:dev-latest" },
            "haproxy": { "image": "perconalab/percona-xtradb-cluster-operator:main-haproxy" },
            "proxysql": { "image": "perconalab/percona-xtradb-cluster-operator:main-proxysql" },
            "logcollector": { "image": "perconalab/fluentbit:main-logcollector" },
            "backup": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup" }
        }}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XcWaNVAI2A
perconaxtradbcluster.pxc.percona.com/upgrade-proxysql patched
+ cat /tmp/tmp.zSaWkpwHn4
+ rm /tmp/tmp.XcWaNVAI2A /tmp/tmp.zSaWkpwHn4
+ return 0
+ sleep 10
+ desc 'check images and generation after full upgrade'
+ set +o xtrace


-----------------------------------------------------------------------------------
check images and generation after full upgrade
-----------------------------------------------------------------------------------

+ check_pxc_liveness upgrade-proxysql 3
+ local cluster=upgrade-proxysql
+ local cluster_size=3
+ wait_cluster_consistency upgrade-proxysql 3
+ local cluster_name=upgrade-proxysql
+ local cluster_size=3
+ local proxy_size=
+ '[' -z '' ']'
++ get_proxy_size upgrade-proxysql
++ local cluster=upgrade-proxysql
+++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.3ExbGy9GGA
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Xt5u5YZeib
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.3ExbGy9GGA
+++ cat /tmp/tmp.Xt5u5YZeib
+++ rm /tmp/tmp.3ExbGy9GGA /tmp/tmp.Xt5u5YZeib
+++ return 0
++ [[ false == \t\r\u\e ]]
+++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Uzn0oJpNbh
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KmkPm55jPQ
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.Uzn0oJpNbh
+++ cat /tmp/tmp.KmkPm55jPQ
+++ rm /tmp/tmp.Uzn0oJpNbh /tmp/tmp.KmkPm55jPQ
+++ return 0
++ [[ true == \t\r\u\e ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.size}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OWiu4qmjvY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5rjYygHeJh
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.size}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.OWiu4qmjvY
++ cat /tmp/tmp.5rjYygHeJh
++ rm /tmp/tmp.OWiu4qmjvY /tmp/tmp.5rjYygHeJh
++ return 0
++ return
+ proxy_size=3
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=300
+ sleep 7
+ echo -n 'waiting for pxc/upgrade-proxysql to be ready'
waiting for pxc/upgrade-proxysql to be ready++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WaSUrVpU0S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yNxnsBlL0z
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WaSUrVpU0S
++ cat /tmp/tmp.yNxnsBlL0z
++ rm /tmp/tmp.WaSUrVpU0S /tmp/tmp.yNxnsBlL0z
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 0 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uNFJyf7lPx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Y9FZLCFUzG
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.uNFJyf7lPx
++ cat /tmp/tmp.Y9FZLCFUzG
++ rm /tmp/tmp.uNFJyf7lPx /tmp/tmp.Y9FZLCFUzG
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 1 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RpGX2Ob6KD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AikVDNWY3V
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RpGX2Ob6KD
++ cat /tmp/tmp.AikVDNWY3V
++ rm /tmp/tmp.RpGX2Ob6KD /tmp/tmp.AikVDNWY3V
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 2 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.soaUvflF8k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Yt20DBpNFb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.soaUvflF8k
++ cat /tmp/tmp.Yt20DBpNFb
++ rm /tmp/tmp.soaUvflF8k /tmp/tmp.Yt20DBpNFb
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 3 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PFEI8lICgl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SFYIcRfxHV
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PFEI8lICgl
++ cat /tmp/tmp.SFYIcRfxHV
++ rm /tmp/tmp.PFEI8lICgl /tmp/tmp.SFYIcRfxHV
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 4 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gUJm0fnBdC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8KJZ5h3gxV
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gUJm0fnBdC
++ cat /tmp/tmp.8KJZ5h3gxV
++ rm /tmp/tmp.gUJm0fnBdC /tmp/tmp.8KJZ5h3gxV
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 5 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.baySWfQKuQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RA8ru6MnUI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.baySWfQKuQ
++ cat /tmp/tmp.RA8ru6MnUI
++ rm /tmp/tmp.baySWfQKuQ /tmp/tmp.RA8ru6MnUI
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 6 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rNq0wXdmfR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4l0bJRjufL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rNq0wXdmfR
++ cat /tmp/tmp.4l0bJRjufL
++ rm /tmp/tmp.rNq0wXdmfR /tmp/tmp.4l0bJRjufL
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 7 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.B0hIgeZ7n4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KdNgrkBRNB
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.B0hIgeZ7n4
++ cat /tmp/tmp.KdNgrkBRNB
++ rm /tmp/tmp.B0hIgeZ7n4 /tmp/tmp.KdNgrkBRNB
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 8 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4G7sxinK6u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WpflCMqszA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4G7sxinK6u
++ cat /tmp/tmp.WpflCMqszA
++ rm /tmp/tmp.4G7sxinK6u /tmp/tmp.WpflCMqszA
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 9 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aZ0BWj3DUg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9nBysJZPH7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.aZ0BWj3DUg
++ cat /tmp/tmp.9nBysJZPH7
++ rm /tmp/tmp.aZ0BWj3DUg /tmp/tmp.9nBysJZPH7
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 10 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bMHXk7gGSF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WrVB5v9ylg
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bMHXk7gGSF
++ cat /tmp/tmp.WrVB5v9ylg
++ rm /tmp/tmp.bMHXk7gGSF /tmp/tmp.WrVB5v9ylg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 11 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2rw6hExA6n
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9v0jnCmN1y
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2rw6hExA6n
++ cat /tmp/tmp.9v0jnCmN1y
++ rm /tmp/tmp.2rw6hExA6n /tmp/tmp.9v0jnCmN1y
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 12 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K38lp0cnpU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6V2QnX5d2I
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.K38lp0cnpU
++ cat /tmp/tmp.6V2QnX5d2I
++ rm /tmp/tmp.K38lp0cnpU /tmp/tmp.6V2QnX5d2I
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 13 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1Gi7cw7aA4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ouX2AHsXrW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1Gi7cw7aA4
++ cat /tmp/tmp.ouX2AHsXrW
++ rm /tmp/tmp.1Gi7cw7aA4 /tmp/tmp.ouX2AHsXrW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 14 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pVcJvcSO8x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Eulq8gKGAq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pVcJvcSO8x
++ cat /tmp/tmp.Eulq8gKGAq
++ rm /tmp/tmp.pVcJvcSO8x /tmp/tmp.Eulq8gKGAq
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 15 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6QlYTk9Ryi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MCE3GYVo7j
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6QlYTk9Ryi
++ cat /tmp/tmp.MCE3GYVo7j
++ rm /tmp/tmp.6QlYTk9Ryi /tmp/tmp.MCE3GYVo7j
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 16 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hCey14Utkl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VJdkWdgz72
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hCey14Utkl
++ cat /tmp/tmp.VJdkWdgz72
++ rm /tmp/tmp.hCey14Utkl /tmp/tmp.VJdkWdgz72
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 17 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KSHNlJlq1s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y6nvCnQ6xU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KSHNlJlq1s
++ cat /tmp/tmp.y6nvCnQ6xU
++ rm /tmp/tmp.KSHNlJlq1s /tmp/tmp.y6nvCnQ6xU
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 18 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GUXS8QlOeu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i0nqlrUtjV
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GUXS8QlOeu
++ cat /tmp/tmp.i0nqlrUtjV
++ rm /tmp/tmp.GUXS8QlOeu /tmp/tmp.i0nqlrUtjV
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 19 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bi09cGYH72
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VeLmM2AmyU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bi09cGYH72
++ cat /tmp/tmp.VeLmM2AmyU
++ rm /tmp/tmp.bi09cGYH72 /tmp/tmp.VeLmM2AmyU
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 20 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2GM7MYL1Nv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xuTkSudeZW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2GM7MYL1Nv
++ cat /tmp/tmp.xuTkSudeZW
++ rm /tmp/tmp.2GM7MYL1Nv /tmp/tmp.xuTkSudeZW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 21 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gjIqhmajDK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QSz56oMLra
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gjIqhmajDK
++ cat /tmp/tmp.QSz56oMLra
++ rm /tmp/tmp.gjIqhmajDK /tmp/tmp.QSz56oMLra
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 22 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZRerr3YP3u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.h2tdPwxrJD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZRerr3YP3u
++ cat /tmp/tmp.h2tdPwxrJD
++ rm /tmp/tmp.ZRerr3YP3u /tmp/tmp.h2tdPwxrJD
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 23 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ginuCNNsIc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JSh7VkqwPu
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ginuCNNsIc
++ cat /tmp/tmp.JSh7VkqwPu
++ rm /tmp/tmp.ginuCNNsIc /tmp/tmp.JSh7VkqwPu
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 24 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PgYL0eREh8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DcJjjb6CT1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PgYL0eREh8
++ cat /tmp/tmp.DcJjjb6CT1
++ rm /tmp/tmp.PgYL0eREh8 /tmp/tmp.DcJjjb6CT1
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 25 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fDiujFST4d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Yd8IweISjA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.fDiujFST4d
++ cat /tmp/tmp.Yd8IweISjA
++ rm /tmp/tmp.fDiujFST4d /tmp/tmp.Yd8IweISjA
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nGmhDdNraq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DQPRyjrwTm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nGmhDdNraq
++ cat /tmp/tmp.DQPRyjrwTm
++ rm /tmp/tmp.nGmhDdNraq /tmp/tmp.DQPRyjrwTm
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine upgrade-proxysql
+++ local cluster_name=upgrade-proxysql
++++ get_proxy upgrade-proxysql
++++ local target_cluster=upgrade-proxysql
+++++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.FJpjYAzoxC
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.dPr4Ah2tii
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.FJpjYAzoxC
+++++ cat /tmp/tmp.dPr4Ah2tii
+++++ rm /tmp/tmp.FJpjYAzoxC /tmp/tmp.dPr4Ah2tii
+++++ return 0
++++ [[ false == \t\r\u\e ]]
+++++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.3M9WjchkvK
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.VsJODOEEcU
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.3M9WjchkvK
+++++ cat /tmp/tmp.VsJODOEEcU
+++++ rm /tmp/tmp.3M9WjchkvK /tmp/tmp.VsJODOEEcU
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo upgrade-proxysql-proxysql
++++ return
+++ local cluster_proxy=upgrade-proxysql-proxysql
+++ echo proxysql
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SBF0KRC1dC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bYGEzWNEhc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SBF0KRC1dC
++ cat /tmp/tmp.bYGEzWNEhc
++ rm /tmp/tmp.SBF0KRC1dC /tmp/tmp.bYGEzWNEhc
++ return 0
+ [[ 3 == \3 ]]
+ echo

+ wait_for_running upgrade-proxysql-pxc 3
+ local name=upgrade-proxysql-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-0 480
+ local pod=upgrade-proxysql-pxc-0
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-0 condition met
waiting for pod/upgrade-proxysql-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-1 480
+ local pod=upgrade-proxysql-pxc-1
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-1 condition met
waiting for pod/upgrade-proxysql-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod upgrade-proxysql-pxc-2 480
+ local pod=upgrade-proxysql-pxc-2
+ local max_retry=480
+ local ns=
++ echo upgrade-proxysql-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/upgrade-proxysql-pxc-2 condition met
waiting for pod/upgrade-proxysql-pxc-2 to become Ready.Ok
++ seq 0 2
+ for i in '$(seq 0 $((cluster_size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-0.upgrade-proxysql-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6yBEXBNBn2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.c6nci2ASyr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6yBEXBNBn2
++ cat /tmp/tmp.c6nci2ASyr
++ rm /tmp/tmp.6yBEXBNBn2 /tmp/tmp.c6nci2ASyr
++ return 0
+ client_pod=pxc-client-cdffbf747-66td6
+ wait_pod pxc-client-cdffbf747-66td6
+ local pod=pxc-client-cdffbf747-66td6
+ local max_retry=480
+ local ns=
++ echo pxc-client-cdffbf747-66td6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-cdffbf747-66td6 condition met
waiting for pod/pxc-client-cdffbf747-66td6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.B2yRMevosb/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.B2yRMevosb/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.B2yRMevosb/select-1.sql
+ for i in '$(seq 0 $((cluster_size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-1.upgrade-proxysql-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OUMb7xytZD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3QbLneBu4d
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.OUMb7xytZD
++ cat /tmp/tmp.3QbLneBu4d
++ rm /tmp/tmp.OUMb7xytZD /tmp/tmp.3QbLneBu4d
++ return 0
+ client_pod=pxc-client-cdffbf747-66td6
+ wait_pod pxc-client-cdffbf747-66td6
+ local pod=pxc-client-cdffbf747-66td6
+ local max_retry=480
+ local ns=
++ echo pxc-client-cdffbf747-66td6
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-cdffbf747-66td6 condition met
waiting for pod/pxc-client-cdffbf747-66td6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.B2yRMevosb/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.B2yRMevosb/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.B2yRMevosb/select-1.sql
+ for i in '$(seq 0 $((cluster_size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h upgrade-proxysql-pxc-2.upgrade-proxysql-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EUIW4pp1lu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J2aDRkRras
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.EUIW4pp1lu
++ cat /tmp/tmp.J2aDRkRras
++ rm /tmp/tmp.EUIW4pp1lu /tmp/tmp.J2aDRkRras
++ return 0
+ client_pod=pxc-client-cdffbf747-66td6
+ wait_pod pxc-client-cdffbf747-66td6
+ local pod=pxc-client-cdffbf747-66td6
+ local max_retry=480
+ local ns=
++ echo pxc-client-cdffbf747-66td6
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-cdffbf747-66td6 condition met
waiting for pod/pxc-client-cdffbf747-66td6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.B2yRMevosb/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.B2yRMevosb/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/select-1.sql /tmp/tmp.B2yRMevosb/select-1.sql
++ kubectl_bin get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dskWJQVwGy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8QOiywxFBj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.dskWJQVwGy
++ cat /tmp/tmp.8QOiywxFBj
++ rm /tmp/tmp.dskWJQVwGy /tmp/tmp.8QOiywxFBj
++ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:PR-2316-524c74d8 == perconalab/percona-xtradb-cluster-operator:PR-2316-524c74d8 ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4lOthsz6Hw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dvv7WTnPsI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.proxysql.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4lOthsz6Hw
++ cat /tmp/tmp.dvv7WTnPsI
++ rm /tmp/tmp.4lOthsz6Hw /tmp/tmp.dvv7WTnPsI
++ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-proxysql == perconalab/percona-xtradb-cluster-operator:main-proxysql ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1JeQiwIJDC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wdXE0qWSVS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.haproxy.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1JeQiwIJDC
++ cat /tmp/tmp.wdXE0qWSVS
++ rm /tmp/tmp.1JeQiwIJDC /tmp/tmp.wdXE0qWSVS
++ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-haproxy == perconalab/percona-xtradb-cluster-operator:main-haproxy ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.backup.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eCd16hpWfJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ehmgdTDK4L
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.backup.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eCd16hpWfJ
++ cat /tmp/tmp.ehmgdTDK4L
++ rm /tmp/tmp.eCd16hpWfJ /tmp/tmp.ehmgdTDK4L
++ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup == perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.pmm.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gFtekvnGSA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bGQsdv2sVA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.pmm.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gFtekvnGSA
++ cat /tmp/tmp.bGQsdv2sVA
++ rm /tmp/tmp.gFtekvnGSA /tmp/tmp.bGQsdv2sVA
++ return 0
+ [[ perconalab/pmm-client:dev-latest == perconalab/pmm-client:dev-latest ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.logcollector.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZKn6kGTTFY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tX8N4agFz3
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.logcollector.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZKn6kGTTFY
++ cat /tmp/tmp.tX8N4agFz3
++ rm /tmp/tmp.ZKn6kGTTFY /tmp/tmp.tX8N4agFz3
++ return 0
+ [[ perconalab/fluentbit:main-logcollector == perconalab/fluentbit:main-logcollector ]]
++ kubectl_bin get pxc upgrade-proxysql -o 'jsonpath={.spec.pxc.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0sgbDoVAyD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fOZumvYppv
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc upgrade-proxysql -o 'jsonpath={.spec.pxc.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0sgbDoVAyD
++ cat /tmp/tmp.fOZumvYppv
++ rm /tmp/tmp.0sgbDoVAyD /tmp/tmp.fOZumvYppv
++ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 == perconalab/percona-xtradb-cluster-operator:main-pxc5.7 ]]
+ : Cluster images have been updated correctly
+ compare_generation 2 proxysql upgrade-proxysql
+ local generation=2
+ local proxy=proxysql
+ local cluster=upgrade-proxysql
+ local current_generation
+ [[ proxysql == \h\a\p\r\o\x\y ]]
+ containers=(pxc proxysql)
+ for container in '"${containers[@]}"'
+ check_generation 2 pxc upgrade-proxysql
+ local generation=2
+ local container=pxc
+ local cluster=upgrade-proxysql
+ local current_generation
++ kubectl_bin get statefulset upgrade-proxysql-pxc -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZWxMRWxtxR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SbVCQpps2k
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset upgrade-proxysql-pxc -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZWxMRWxtxR
++ cat /tmp/tmp.SbVCQpps2k
++ rm /tmp/tmp.ZWxMRWxtxR /tmp/tmp.SbVCQpps2k
++ return 0
+ current_generation=2
+ [[ 2 != \2 ]]
+ for container in '"${containers[@]}"'
+ check_generation 2 proxysql upgrade-proxysql
+ local generation=2
+ local container=proxysql
+ local cluster=upgrade-proxysql
+ local current_generation
++ kubectl_bin get statefulset upgrade-proxysql-proxysql -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nHpVgZM57s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nDNEXxmSZ3
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset upgrade-proxysql-proxysql -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nHpVgZM57s
++ cat /tmp/tmp.nDNEXxmSZ3
++ rm /tmp/tmp.nHpVgZM57s /tmp/tmp.nDNEXxmSZ3
++ return 0
+ current_generation=2
+ [[ 2 != \2 ]]
+ compare_kubectl statefulset/upgrade-proxysql-pxc
+ local resource=statefulset/upgrade-proxysql-pxc
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc.yml
+ local new_result=/tmp/tmp.B2yRMevosb/statefulset_upgrade-proxysql-pxc.yml
+ desc 'compare statefulset/upgrade-proxysql-pxc-'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/upgrade-proxysql-pxc-
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-eks.yml ']'
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ version_gt 1.33
++ echo '1.31 >= 1.33'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.29
++ echo '1.31 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-k129.yml ']'
+ version_gt 1.27
++ bc -l
++ echo '1.31 >= 1.27'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-k127.yml ']'
+ version_gt 1.24
++ bc -l
++ echo '1.31 >= 1.24'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-k124.yml ']'
+ version_gt 1.22
++ bc -l
++ echo '1.31 >= 1.22'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-k122.yml ']'
+ version_gt 1.21
++ bc -l
++ echo '1.31 >= 1.21'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-oc.yml ']'
+ version_gt 1.29
++ echo '1.31 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-k129-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-eks.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc-aks.yml ']'
+ kubectl_bin get -o yaml statefulset/upgrade-proxysql-pxc
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.labels."batch.kubernetes.io/job-name") |
			del(.metadata.labels."job-name") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") |
			del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") |
			del(.spec.template.metadata.labels."job-name") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-proxysql-2190", "namespace") |
			(.. |++ mktemp
 select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.spec.crVersion) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.inM8WQ3JSC
++ mktemp
+ local LAST_ERR=/tmp/tmp.LS9EPKktri
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/upgrade-proxysql-pxc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.inM8WQ3JSC
+ cat /tmp/tmp.LS9EPKktri
+ rm /tmp/tmp.inM8WQ3JSC /tmp/tmp.LS9EPKktri
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-pxc.yml /tmp/tmp.B2yRMevosb/statefulset_upgrade-proxysql-pxc.yml
+ log 'compare_kubectl: statefulset/upgrade-proxysql-pxc OK'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2025-12-18T21:04:25+0000]' compare_kubectl: statefulset/upgrade-proxysql-pxc OK
[2025-12-18T21:04:25+0000] compare_kubectl: statefulset/upgrade-proxysql-pxc OK
+ compare_kubectl statefulset/upgrade-proxysql-proxysql
+ local resource=statefulset/upgrade-proxysql-proxysql
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql.yml
+ local new_result=/tmp/tmp.B2yRMevosb/statefulset_upgrade-proxysql-proxysql.yml
+ desc 'compare statefulset/upgrade-proxysql-proxysql-'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/upgrade-proxysql-proxysql-
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-eks.yml ']'
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]]
+ version_gt 1.33
++ echo '1.31 >= 1.33'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.29
++ echo '1.31 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-k129.yml ']'
+ version_gt 1.27
++ echo '1.31 >= 1.27'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-k127.yml ']'
+ version_gt 1.24
++ echo '1.31 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-k124.yml ']'
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-k122.yml ']'
+ version_gt 1.21
++ echo '1.31 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-oc.yml ']'
+ version_gt 1.29
++ echo '1.31 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-k129-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-eks.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql-aks.yml ']'
+ kubectl_bin get -o yaml statefulset/upgrade-proxysql-proxysql
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.labels."batch.kubernetes.io/job-name") |
			del(.metadata.labels."job-name") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") |
			del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") |
			del(.spec.template.metadata.labels."job-name") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-proxysql-2190", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.spec.crVersion) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.VhAU9UV58x
++ mktemp
+ local LAST_ERR=/tmp/tmp.eIPAHgL1nB
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/upgrade-proxysql-proxysql
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.VhAU9UV58x
+ cat /tmp/tmp.eIPAHgL1nB
+ rm /tmp/tmp.VhAU9UV58x /tmp/tmp.eIPAHgL1nB
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2316/e2e-tests/upgrade-proxysql/compare/statefulset_upgrade-proxysql-proxysql.yml /tmp/tmp.B2yRMevosb/statefulset_upgrade-proxysql-proxysql.yml
+ log 'compare_kubectl: statefulset/upgrade-proxysql-proxysql OK'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2025-12-18T21:04:26+0000]' compare_kubectl: statefulset/upgrade-proxysql-proxysql OK
[2025-12-18T21:04:26+0000] compare_kubectl: statefulset/upgrade-proxysql-proxysql OK
+ desc cleanup
+ set +o xtrace


-----------------------------------------------------------------------------------
cleanup
-----------------------------------------------------------------------------------

+ destroy upgrade-proxysql-2190
+ local namespace=upgrade-proxysql-2190
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
+ grep -v level=info
+ grep -v 'get backup status: Job.batch'
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ sort -u
+ tee /tmp/tmp.B2yRMevosb/operator.log
++ get_operator_pod
+ grep -v 'the object has been modified'
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ head -1
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xHyj3c3YzD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Oh3H3N3vuI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xHyj3c3YzD
++ cat /tmp/tmp.Oh3H3N3vuI
++ rm /tmp/tmp.xHyj3c3YzD /tmp/tmp.Oh3H3N3vuI
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-7d57c5dcc9-cnqc8
++ mktemp
+ local LAST_OUT=/tmp/tmp.mJJ43aY6aQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.Kde2DugE2w
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-7d57c5dcc9-cnqc8
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.mJJ43aY6aQ
+ cat /tmp/tmp.Kde2DugE2w
+ rm /tmp/tmp.mJJ43aY6aQ /tmp/tmp.Kde2DugE2w
+ return 0
2025-12-18T20:59:05.367Z	INFO	setup	Feature gates	{"PXCO_FEATURE_GATES": "", "enabled": ""}
2025-12-18T20:59:05.367Z	INFO	setup	Manager starting up	{"gitCommit": "524c74d8f42d46f153bdf7f5f53ae9a845b2cb0c", "gitBranch": "PR-2316-524c74d8", "buildTime": "2025-12-18T17:55:45Z", "goVersion": "go1.25.5", "os": "linux", "arch": "amd64"}
2025-12-18T20:59:05.367Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.31.14-gke.1081000"}
2025-12-18T20:59:05.370Z	INFO	setup	Registering Components.
2025-12-18T20:59:06.363Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2025-12-18T20:59:06.363Z	INFO	controller-runtime.metrics	Starting metrics server
2025-12-18T20:59:06.363Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2025-12-18T20:59:06.363Z	INFO	controller-runtime.webhook	Starting webhook server
2025-12-18T20:59:06.363Z	INFO	setup	Starting the Cmd.
2025-12-18T20:59:06.363Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2025-12-18T20:59:06.364Z	INFO	controller-runtime.certwatcher	Starting certificate poll+watcher	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"}
2025-12-18T20:59:06.364Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"}
2025-12-18T20:59:06.364Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2025-12-18T20:59:06.564Z	INFO	Attempting to acquire leader lease...	{"lock": "pxc-operator/08db1feb.percona.com"}
2025-12-18T20:59:32.981Z	DEBUG	events	percona-xtradb-cluster-operator-7d57c5dcc9-cnqc8_e352b35a-8e99-4f1f-a723-db379bdc3c7a became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"1635aa1f-731c-4136-876d-739d72980446","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1766091572976351009"}, "reason": "LeaderElection"}
2025-12-18T20:59:32.981Z	INFO	Successfully acquired lease	{"lock": "pxc-operator/08db1feb.percona.com"}
2025-12-18T20:59:32.982Z	INFO	Starting EventSource	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"}
2025-12-18T20:59:32.983Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2025-12-18T20:59:32.983Z	INFO	Starting EventSource	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"}
2025-12-18T20:59:32.983Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2025-12-18T20:59:33.083Z	INFO	Starting Controller	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"}
2025-12-18T20:59:33.083Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1}
2025-12-18T20:59:33.084Z	INFO	Starting Controller	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"}
2025-12-18T20:59:33.084Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1}
2025-12-18T20:59:33.085Z	INFO	Starting Controller	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"}
2025-12-18T20:59:33.085Z	INFO	Starting workers	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1}
2025-12-18T20:59:33.463Z	DEBUG	Updating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "001f5015-6588-40cc-980e-b1abfe630e28", "object": "upgrade-proxysql-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-12-18T20:59:33.597Z	DEBUG	Updating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "001f5015-6588-40cc-980e-b1abfe630e28", "object": "upgrade-proxysql-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}", "hashChanged": true, "metaChanged": true}
2025-12-18T20:59:33.619Z	DEBUG	Updating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "001f5015-6588-40cc-980e-b1abfe630e28", "object": "upgrade-proxysql-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-12-18T20:59:33.661Z	DEBUG	Updating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "001f5015-6588-40cc-980e-b1abfe630e28", "object": "upgrade-proxysql-proxysql", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}", "hashChanged": true, "metaChanged": true}
2025-12-18T20:59:34.716Z	INFO	Creating or updating backup job	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "001f5015-6588-40cc-980e-b1abfe630e28", "name": "acb27-daily-backup", "schedule": "0 0 * * *"}
2025-12-18T20:59:35.290Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "001f5015-6588-40cc-980e-b1abfe630e28"}
2025-12-18T20:59:41.967Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "aa1ad752-87f2-4c84-bd50-4444c214b25a"}
2025-12-18T20:59:48.190Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "68fba3be-80c9-4dba-bc03-f367275a9fef"}
2025-12-18T20:59:54.347Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "110de543-afdd-405b-8799-557c478d2158"}
2025-12-18T21:00:00.474Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "1556260d-5d35-46fb-9ac2-b05fc036410a"}
2025-12-18T21:00:06.249Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "ec337491-1ad1-4cf2-abe7-b4624034e5d8"}
2025-12-18T21:00:13.112Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "6d3e5bd1-c98d-4190-9030-91d0ecbfa769"}
2025-12-18T21:00:19.072Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "ceac93eb-b34d-4f3b-a1fd-8a31efa147b6"}
2025-12-18T21:00:25.543Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "fc5d25d5-a441-41de-8f05-661d6ffd7742"}
2025-12-18T21:00:27.351Z	DEBUG	Updating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "object": "upgrade-proxysql-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-12-18T21:00:27.407Z	DEBUG	Updating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "object": "upgrade-proxysql-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-12-18T21:00:28.478Z	INFO	statefulSet was changed, run smart update	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e"}
2025-12-18T21:00:28.483Z	INFO	apply changes to secondary pod	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-2"}
2025-12-18T21:00:28.483Z	INFO	primary pod	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-0.upgrade-proxysql-pxc.upgrade-proxysql-2190.svc.cluster.local"}
2025-12-18T21:00:29.179Z	INFO	Pod is not updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-2", "currentRevision": "upgrade-proxysql-pxc-96df78c5d", "targetRevision": "upgrade-proxysql-pxc-7d4f547d5b"}
2025-12-18T21:00:29.694Z	ERROR	sync users	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "error": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") /  / ", "errorVerbose": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") /  / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:975\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:855\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2025-12-18T21:00:39.297Z	INFO	Pod is not updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-2", "currentRevision": "upgrade-proxysql-pxc-96df78c5d", "targetRevision": "upgrade-proxysql-pxc-7d4f547d5b"}
2025-12-18T21:01:19.289Z	INFO	Pod is updated, running and ready	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-2"}
2025-12-18T21:01:19.308Z	INFO	waiting for pod to be present in hostgroup	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-2", "host": "upgrade-proxysql-pxc-2.upgrade-proxysql-pxc.upgrade-proxysql-2190", "hostgroup": "10"}
2025-12-18T21:01:29.311Z	INFO	waiting for pod to be present in hostgroup	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-2", "host": "upgrade-proxysql-pxc-2.upgrade-proxysql-pxc.upgrade-proxysql-2190", "hostgroup": "10"}
2025-12-18T21:01:39.310Z	INFO	pod present in hostgroup	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-2", "host": "upgrade-proxysql-pxc-2.upgrade-proxysql-pxc.upgrade-proxysql-2190", "hostgroup": "10"}
2025-12-18T21:01:39.314Z	INFO	apply changes to secondary pod	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-1"}
2025-12-18T21:01:39.314Z	INFO	pod is online	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-2"}
2025-12-18T21:01:39.484Z	INFO	Pod is not updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-1", "currentRevision": "upgrade-proxysql-pxc-96df78c5d", "targetRevision": "upgrade-proxysql-pxc-7d4f547d5b"}
2025-12-18T21:01:49.587Z	INFO	Pod is not updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-1", "currentRevision": "upgrade-proxysql-pxc-96df78c5d", "targetRevision": "upgrade-proxysql-pxc-7d4f547d5b"}
2025-12-18T21:01:59.563Z	INFO	Pod is not running	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-1", "phase": "Pending"}
2025-12-18T21:01:59.563Z	INFO	pod is waiting	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-1", "reason": "PodInitializing"}
2025-12-18T21:02:29.596Z	INFO	Pod is updated, running and ready	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-1"}
2025-12-18T21:02:29.611Z	INFO	waiting for pod to be present in hostgroup	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-1", "host": "upgrade-proxysql-pxc-1.upgrade-proxysql-pxc.upgrade-proxysql-2190", "hostgroup": "10"}
2025-12-18T21:02:39.613Z	INFO	pod present in hostgroup	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-1", "host": "upgrade-proxysql-pxc-1.upgrade-proxysql-pxc.upgrade-proxysql-2190", "hostgroup": "10"}
2025-12-18T21:02:39.616Z	INFO	pod is online	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-1"}
2025-12-18T21:02:39.617Z	INFO	apply changes to primary pod	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-0"}
2025-12-18T21:02:39.793Z	INFO	Pod is not updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-0", "currentRevision": "upgrade-proxysql-pxc-96df78c5d", "targetRevision": "upgrade-proxysql-pxc-7d4f547d5b"}
2025-12-18T21:02:49.931Z	INFO	Pod is not updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-0", "currentRevision": "upgrade-proxysql-pxc-96df78c5d", "targetRevision": "upgrade-proxysql-pxc-7d4f547d5b"}
2025-12-18T21:02:59.866Z	INFO	Pod is not running	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-0", "phase": "Pending"}
2025-12-18T21:02:59.866Z	INFO	pod is waiting	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-0", "reason": "PodInitializing"}
2025-12-18T21:03:29.934Z	INFO	Pod is updated, running and ready	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-0"}
2025-12-18T21:03:29.954Z	INFO	pod present in hostgroup	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-0", "host": "upgrade-proxysql-pxc-0.upgrade-proxysql-pxc.upgrade-proxysql-2190", "hostgroup": "11"}
2025-12-18T21:03:29.957Z	INFO	pod is online	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "pod": "upgrade-proxysql-pxc-0"}
2025-12-18T21:03:29.957Z	INFO	smart update finished	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e"}
2025-12-18T21:03:29.997Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "12b708f4-cfe2-4c0f-890c-16523427186e", "new version": "5.7.44-48-57"}
2025-12-18T21:03:32.569Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "0dadac58-361d-42d0-a65b-58915c150345"}
2025-12-18T21:03:37.553Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "67233f0b-8225-415b-96db-10ede77ffc7a"}
2025-12-18T21:03:43.823Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "0ec5c1de-e27f-4aea-8c02-b91bd45223b8"}
2025-12-18T21:03:49.977Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "44e16a96-c2c9-45a1-a875-d18f44dcfb81"}
2025-12-18T21:03:56.212Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "831aaf8e-02a7-4398-b146-c506b91ece43"}
2025-12-18T21:04:02.679Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "b97ee044-d176-4d7e-bb13-8f0dfa3521c6"}
2025-12-18T21:04:07.877Z	ERROR	sync users	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "f41a54f2-061a-41dd-ac51-b3da285b76a2", "error": "syncusers: ERROR (line:1301) : Could not find any nodes belonging to the cluster with writer hostgroup:11\nERROR (line:572) : Could not find a primary cluster node\n", "errorVerbose": "syncusers: ERROR (line:1301) : Could not find any nodes belonging to the cluster with writer hostgroup:11\nERROR (line:572) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:979\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:855\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2025-12-18T21:04:15.367Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "4c86555c-5ffd-4f03-aa07-a9120533a259"}
2025-12-18T21:04:21.551Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "2786c68f-f4a5-400b-9a80-7589c2bd05da"}
2025-12-18T21:04:27.960Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"upgrade-proxysql","namespace":"upgrade-proxysql-2190"}, "namespace": "upgrade-proxysql-2190", "name": "upgrade-proxysql", "reconcileID": "2f5420e4-3549-4f50-aa36-a7105541a755"}
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:857
+ grep -v NAMESPACE
+ kubectl get pxc --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n upgrade-proxysql-2190 upgrade-proxysql --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/upgrade-proxysql patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.GjO7OTNDjo
++ mktemp
+ local LAST_ERR=/tmp/tmp.UHylVlIYBt
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.GjO7OTNDjo
perconaxtradbcluster.pxc.percona.com "upgrade-proxysql" deleted from upgrade-proxysql-2190 namespace
+ cat /tmp/tmp.UHylVlIYBt
+ rm /tmp/tmp.GjO7OTNDjo /tmp/tmp.UHylVlIYBt
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.jFq7bq6uik
++ mktemp
+ local LAST_ERR=/tmp/tmp.bLKsRbdCxT
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.jFq7bq6uik
No resources found
+ cat /tmp/tmp.bLKsRbdCxT
+ rm /tmp/tmp.jFq7bq6uik /tmp/tmp.bLKsRbdCxT
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.ntAv6rujdq
++ mktemp
+ local LAST_ERR=/tmp/tmp.Hl4ARoeFNF
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ntAv6rujdq
No resources found
+ cat /tmp/tmp.Hl4ARoeFNF
+ rm /tmp/tmp.ntAv6rujdq /tmp/tmp.Hl4ARoeFNF
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.6RgK5yK5Ny
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZlZ32VzbXb
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6RgK5yK5Ny
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.ZlZ32VzbXb
+ rm /tmp/tmp.6RgK5yK5Ny /tmp/tmp.ZlZ32VzbXb
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ rm -rf /tmp/tmp.B2yRMevosb
+ kubectl_bin delete --grace-period=0 --force=true namespace upgrade-proxysql-2190
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ne1TiuyUJs
+ local LAST_OUT=/tmp/tmp.bjcHkw73fT
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.7fiys1odoz
+ local exit_status=0
++ kubectl get crd
+ local LAST_ERR=/tmp/tmp.UI1hxrFp12
+ local exit_status=0
++ seq 0 2
++ tr '\n' ' '
++ grep perconaxtradbcluster
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace upgrade-proxysql-2190
++ awk '{print $1}'
+ kubectl delete crd perconaxtradbclusterbackups.pxc.percona.com perconaxtradbclusterrestores.pxc.percona.com perconaxtradbclusters.pxc.percona.com
customresourcedefinition.apiextensions.k8s.io "perconaxtradbclusterbackups.pxc.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaxtradbclusterrestores.pxc.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaxtradbclusters.pxc.percona.com" deleted
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------