Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/logs/upgrade-haproxy-5-7.log Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 + CLUSTER=upgrade-haproxy + CLUSTER_SIZE=3 + TARGET_OPERATOR_VER=1.20.0 + TARGET_IMAGE=perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc5.7 + TARGET_IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest + TARGET_IMAGE_PROXY=perconalab/percona-xtradb-cluster-operator:main-proxysql + TARGET_IMAGE_HAPROXY=perconalab/percona-xtradb-cluster-operator:main-haproxy + TARGET_IMAGE_BACKUP=perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup + TARGET_IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc5.7 ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' + PXC_VER=5.7 ++ curl -s https://check.percona.com/versions/v1/pxc-operator ++ jq -r '.versions[].operator' ++ sort -V ++ tail -n1 + INIT_OPERATOR_VER=1.19.0 + [[ 1.19.0 == \1\.\2\0\.\0 ]] + GIT_TAG=v1.19.0 ++ curl -s 'https://check.percona.com/versions/v1/pxc-operator/1.19.0/latest?databaseVersion=5.7' + INIT_OPERATOR_IMAGES='{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}' + OPERATOR_NAME=percona-xtradb-cluster-operator ++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}' ++ jq -r '.versions[].matrix.operator[].imagePath' + IMAGE=percona/percona-xtradb-cluster-operator:1.19.0 ++ echo perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd ++ cut -d/ -f1 + [[ perconalab == \p\e\r\c\o\n\a\l\a\b ]] + IMAGE=perconalab/percona-xtradb-cluster-operator:1.19.0 ++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}' ++ jq -r '.versions[].matrix.pxc[].imagePath' + IMAGE_PXC=percona/percona-xtradb-cluster:5.7.44-31.65 ++ jq -r '.versions[].matrix.pmm[].imagePath' ++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}' ++ tail -n1 + IMAGE_PMM_CLIENT=percona/pmm-client:3.5.0 ++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}' ++ jq -r '.versions[].matrix.proxysql[].imagePath' + IMAGE_PROXY=percona/proxysql2:2.7.3-1.2 ++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}' ++ jq -r '.versions[].matrix.haproxy[].imagePath' + IMAGE_HAPROXY=percona/haproxy:2.8.17 ++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}' ++ jq -r '.versions[].matrix.backup[].imagePath' + IMAGE_BACKUP=percona/percona-xtrabackup:2.4.29 ++ echo '{"versions":[{"product":"pxc-operator","operator":"1.19.0","matrix":{"mongod":{},"pxc":{"5.7.44-31.65":{"imagePath":"percona/percona-xtradb-cluster:5.7.44-31.65","imageHash":"36fafdef46485839d4ff7c6dc73b4542b07031644c0152e911acb9734ff2be85","imageHashArm64":"","status":"recommended","critical":false}},"pmm":{"2.44.1-1":{"imagePath":"percona/pmm-client:2.44.1-1","imageHash":"52a8fb5e8f912eef1ff8a117ea323c401e278908ce29928dafc23fac1db4f1e3","imageHashArm64":"390bfd12f981e8b3890550c4927a3ece071377065e001894458047602c744e3b","status":"recommended","critical":false},"3.5.0":{"imagePath":"percona/pmm-client:3.5.0","imageHash":"352aee74f25b3c1c4cd9dff1f378a0c3940b315e551d170c09953bf168531e4a","imageHashArm64":"cbbb074d51d90a5f2d6f1d98a05024f6de2ffdcb5acab632324cea4349a820bd","status":"recommended","critical":false}},"proxysql":{"2.7.3-1.2":{"imagePath":"percona/proxysql2:2.7.3-1.2","imageHash":"719d0ab363c65c7f75431bbed7ec0d9f2af7e691765c489da954813c552359a2","imageHashArm64":"4c4d094652c9f2eb097be5d92dcc05da61c9e8699ac7321def959d5a205a89f7","status":"recommended","critical":false}},"haproxy":{"2.8.17":{"imagePath":"percona/haproxy:2.8.17","imageHash":"ef8486b39a1e8dca97b5cdf1135e6282be1757ad188517b889d12c5a3470eeda","imageHashArm64":"bbc5b3b66ac985d1a4500195539e7dff5196245a5a842a6858ea0848ec089967","status":"recommended","critical":false}},"backup":{"2.4.29":{"imagePath":"percona/percona-xtrabackup:2.4.29","imageHash":"11b92a7f7362379fc6b0de92382706153f2ac007ebf0d7ca25bac2c7303fdf10","imageHashArm64":"","status":"recommended","critical":false}},"operator":{"1.19.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.19.0","imageHash":"6ccbac5e74f5b5309fd4788c5b8d91d5abd01850a4a356ad9eff9f82d20afb51","imageHashArm64":"1ed2a5ab22ee7588aa17ec2339876dc72c9724dc9a81506ff449a2b1aa085024","status":"recommended","critical":false}},"logCollector":{"4.0.1-1":{"imagePath":"percona/fluentbit:4.0.1-1","imageHash":"65bdf7d38cbceed6b6aa6412aea3fb4a196000ac6c66185f114a0a62c4a442ad","imageHashArm64":"dabda77b298b67d30d7f53b5cdb7215ad19dabb22b9543e3fd8aedb74ab24733","status":"recommended","critical":false}},"postgresql":{},"pgbackrest":{},"pgbackrestRepo":{},"pgbadger":{},"pgbouncer":{},"pxcOperator":{},"psmdbOperator":{},"pgOperatorApiserver":{},"pgOperatorEvent":{},"pgOperatorRmdata":{},"pgOperatorScheduler":{},"pgOperator":{},"pgOperatorDeployer":{},"psOperator":{},"mysql":{},"router":{},"orchestrator":{},"toolkit":{},"postgis":{}}}]}' ++ jq -r '.versions[].matrix.logCollector[].imagePath' + IMAGE_LOGCOLLECTOR=percona/fluentbit:4.0.1-1 + [[ -n '' ]] + [[ 1.20.0 == \1\.\1\9\.\0 ]] + main + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.JzuJpnIiNE ++ mktemp + local LAST_ERR=/tmp/tmp.dgNfpUxUBt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JzuJpnIiNE namespace/cert-manager created + cat /tmp/tmp.dgNfpUxUBt + rm /tmp/tmp.JzuJpnIiNE /tmp/tmp.dgNfpUxUBt + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.LkXx5BVBzH ++ mktemp + local LAST_ERR=/tmp/tmp.nSN6GfGckp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LkXx5BVBzH namespace/cert-manager labeled + cat /tmp/tmp.nSN6GfGckp + rm /tmp/tmp.LkXx5BVBzH /tmp/tmp.nSN6GfGckp + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.hMPEssTl7f ++ mktemp + local LAST_ERR=/tmp/tmp.iSfZI5nRzL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hMPEssTl7f namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.iSfZI5nRzL Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.hMPEssTl7f /tmp/tmp.iSfZI5nRzL + return 0 + '[' '' == 4.10 ']' + sleep 70 + create_infra_gh upgrade-haproxy-28003 v1.19.0 + local ns=upgrade-haproxy-28003 + local git_tag=v1.19.0 + '[' -n pxc-operator ']' + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.l5lpZcqMPU + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.vxPuhiGk8k + local exit_status=0 + local LAST_OUT=/tmp/tmp.xiySz2YNVo ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator ++ mktemp + local LAST_ERR=/tmp/tmp.EMGKNOKuqJ + local exit_status=0 ++ seq 0 2 + xargs kubectl delete ns + for i in '$(seq 0 2)' + set +e + kubectl get ns + awk '{print$1}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xiySz2YNVo + cat /tmp/tmp.EMGKNOKuqJ + rm /tmp/tmp.xiySz2YNVo /tmp/tmp.EMGKNOKuqJ + return 0 namespace "cert-manager" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.l5lpZcqMPU namespace "pxc-operator" deleted + cat /tmp/tmp.vxPuhiGk8k + rm /tmp/tmp.l5lpZcqMPU /tmp/tmp.vxPuhiGk8k + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.DcGHuiyPRZ ++ mktemp + local LAST_ERR=/tmp/tmp.P41lO2FtMG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DcGHuiyPRZ namespace/pxc-operator created + cat /tmp/tmp.P41lO2FtMG + rm /tmp/tmp.DcGHuiyPRZ /tmp/tmp.P41lO2FtMG + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.XzRkpKJfCd +++ mktemp ++ local LAST_ERR=/tmp/tmp.ApnxwZQ86f ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XzRkpKJfCd ++ cat /tmp/tmp.ApnxwZQ86f ++ rm /tmp/tmp.XzRkpKJfCd /tmp/tmp.ApnxwZQ86f ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster3 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.XRYIkgekRg ++ mktemp + local LAST_ERR=/tmp/tmp.WYqHLeonZQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster3 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XRYIkgekRg Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster3" modified. + cat /tmp/tmp.WYqHLeonZQ + rm /tmp/tmp.XRYIkgekRg /tmp/tmp.WYqHLeonZQ + return 0 + deploy_operator_gh v1.19.0 + local git_tag=v1.19.0 + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- ++ kubectl_bin get crds -o 'jsonpath={.items[?(@.metadata.name == "perconaxtradbclusters.pxc.percona.com")].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JdGqhsjdE3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.TlIJ6Jf8ld ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crds -o 'jsonpath={.items[?(@.metadata.name == "perconaxtradbclusters.pxc.percona.com")].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JdGqhsjdE3 ++ cat /tmp/tmp.TlIJ6Jf8ld ++ rm /tmp/tmp.JdGqhsjdE3 /tmp/tmp.TlIJ6Jf8ld ++ return 0 + [[ -n perconaxtradbclusters.pxc.percona.com ]] ++ kubectl_bin get crd/perconaxtradbclusters.pxc.percona.com -o 'jsonpath={.spec.versions[?(@.name == "v1-19-0")].name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1lDk0WKglJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.qAgNKLwbtd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/perconaxtradbclusters.pxc.percona.com -o 'jsonpath={.spec.versions[?(@.name == "v1-19-0")].name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1lDk0WKglJ ++ cat /tmp/tmp.qAgNKLwbtd ++ rm /tmp/tmp.1lDk0WKglJ /tmp/tmp.qAgNKLwbtd ++ return 0 + [[ -n '' ]] + kubectl_bin apply --server-side --force-conflicts -f https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.19.0/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.9axNc1KvhM ++ mktemp + local LAST_ERR=/tmp/tmp.msE7GTaEDb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.19.0/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9axNc1KvhM + cat /tmp/tmp.msE7GTaEDb + rm /tmp/tmp.9axNc1KvhM /tmp/tmp.msE7GTaEDb + return 0 + local rbac_yaml=rbac + local operator_yaml=operator.yaml + '[' -n pxc-operator ']' + rbac_yaml=cw-rbac + operator_yaml=cw-operator.yaml + apply_rbac_gh cw-rbac v1.19.0 + local operator_namespace=pxc-operator + local rbac=cw-rbac + local git_tag=v1.19.0 + curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.19.0/deploy/cw-rbac.yaml + /usr/bin/sed -i -e 's^namespace: .*^namespace: pxc-operator^' /tmp/tmp.Zvl3zkw7nj/rbac_v1.19.0.yaml + kubectl_bin apply -f /tmp/tmp.Zvl3zkw7nj/rbac_v1.19.0.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.wzZKfGgPxn ++ mktemp + local LAST_ERR=/tmp/tmp.WUvZBAPoiM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /tmp/tmp.Zvl3zkw7nj/rbac_v1.19.0.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wzZKfGgPxn clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.WUvZBAPoiM + rm /tmp/tmp.wzZKfGgPxn /tmp/tmp.WUvZBAPoiM + return 0 + curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.19.0/deploy/cw-operator.yaml + cat /tmp/tmp.Zvl3zkw7nj/cw-operator.yaml_v1.19.0.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:1.19.0^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' + kubectl_bin apply -n pxc-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.Tcvb7juyyh ++ mktemp + local LAST_ERR=/tmp/tmp.NfITap2dL4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n pxc-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Tcvb7juyyh deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.NfITap2dL4 + rm /tmp/tmp.Tcvb7juyyh /tmp/tmp.NfITap2dL4 + return 0 + sleep 10 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ mktemp ++ local LAST_OUT=/tmp/tmp.e4NgbartEp +++ mktemp ++ local LAST_ERR=/tmp/tmp.o4ql6onTvI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.e4NgbartEp ++ cat /tmp/tmp.o4ql6onTvI ++ rm /tmp/tmp.e4NgbartEp /tmp/tmp.o4ql6onTvI ++ return 0 + wait_pod percona-xtradb-cluster-operator-7f865c95d-z8dld + local pod=percona-xtradb-cluster-operator-7f865c95d-z8dld + local max_retry=480 + local ns= ++ echo percona-xtradb-cluster-operator-7f865c95d-z8dld ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-7f865c95d-z8dld condition met waiting for pod/percona-xtradb-cluster-operator-7f865c95d-z8dld to become Ready.Ok + create_namespace upgrade-haproxy-28003 + local namespace=upgrade-haproxy-28003 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrole ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces upgrade-haproxy-28003' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces upgrade-haproxy-28003 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace upgrade-haproxy-28003 + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.tZRcG4wTGD ++ mktemp + kubectl_bin get ns + local LAST_ERR=/tmp/tmp.5wEMJLcDZC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace upgrade-haproxy-28003 + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.cMZfrDUgfN ++ mktemp + local LAST_ERR=/tmp/tmp.12aZFxnhUO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace upgrade-haproxy-28003 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cMZfrDUgfN + cat /tmp/tmp.12aZFxnhUO + rm /tmp/tmp.cMZfrDUgfN /tmp/tmp.12aZFxnhUO + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace upgrade-haproxy-28003 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.tZRcG4wTGD + cat /tmp/tmp.5wEMJLcDZC Error from server (NotFound): namespaces "upgrade-haproxy-28003" not found + rm /tmp/tmp.tZRcG4wTGD /tmp/tmp.5wEMJLcDZC + return 1 + : + wait_for_delete namespace/upgrade-haproxy-28003 + local res=namespace/upgrade-haproxy-28003 + echo -n 'waiting for namespace/upgrade-haproxy-28003 to be deleted' waiting for namespace/upgrade-haproxy-28003 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "upgrade-haproxy-28003" not found + desc 'create namespace upgrade-haproxy-28003' + set +o xtrace ----------------------------------------------------------------------------------- create namespace upgrade-haproxy-28003 ----------------------------------------------------------------------------------- + kubectl_bin create namespace upgrade-haproxy-28003 ++ mktemp + local LAST_OUT=/tmp/tmp.zRBb5bpBsJ ++ mktemp + local LAST_ERR=/tmp/tmp.01WIm8QeD6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace upgrade-haproxy-28003 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zRBb5bpBsJ namespace/upgrade-haproxy-28003 created + cat /tmp/tmp.01WIm8QeD6 + rm /tmp/tmp.zRBb5bpBsJ /tmp/tmp.01WIm8QeD6 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z2NIgSarKA +++ mktemp ++ local LAST_ERR=/tmp/tmp.Slk2ilrYDZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Z2NIgSarKA ++ cat /tmp/tmp.Slk2ilrYDZ ++ rm /tmp/tmp.Z2NIgSarKA /tmp/tmp.Slk2ilrYDZ ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster3 --namespace=upgrade-haproxy-28003 ++ mktemp + local LAST_OUT=/tmp/tmp.ATCyuXJKO2 ++ mktemp + local LAST_ERR=/tmp/tmp.nPb03LqVzF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster3 --namespace=upgrade-haproxy-28003 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ATCyuXJKO2 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster3" modified. + cat /tmp/tmp.nPb03LqVzF + rm /tmp/tmp.ATCyuXJKO2 /tmp/tmp.nPb03LqVzF + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.UnOI75mZz5 ++ mktemp + local LAST_ERR=/tmp/tmp.t2hIwM8ebk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UnOI75mZz5 secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.t2hIwM8ebk + rm /tmp/tmp.UnOI75mZz5 /tmp/tmp.t2hIwM8ebk + return 0 + start_minio + deploy_helm upgrade-haproxy-28003 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "chaos-mesh" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Wed Apr 8 18:48:27 2026 NAMESPACE: upgrade-haproxy-28003 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.upgrade-haproxy-28003.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace upgrade-haproxy-28003 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace upgrade-haproxy-28003 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace upgrade-haproxy-28003 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace upgrade-haproxy-28003 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CjczVYEbn0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.mZH9TYSPkU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CjczVYEbn0 ++ cat /tmp/tmp.mZH9TYSPkU ++ rm /tmp/tmp.CjczVYEbn0 /tmp/tmp.mZH9TYSPkU ++ return 0 + MINIO_POD=minio-service-5fd5489bdc-r9msl + wait_pod minio-service-5fd5489bdc-r9msl + local pod=minio-service-5fd5489bdc-r9msl + local max_retry=480 + local ns= ++ echo minio-service-5fd5489bdc-r9msl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minio-service-5fd5489bdc-r9msl condition met waiting for pod/minio-service-5fd5489bdc-r9msl to become Ready.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.5dDd3GDdI8 ++ mktemp + local LAST_ERR=/tmp/tmp.onJyvZJfN7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5dDd3GDdI8 pod "aws-cli" deleted from upgrade-haproxy-28003 namespace + cat /tmp/tmp.onJyvZJfN7 All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.5dDd3GDdI8 /tmp/tmp.onJyvZJfN7 + return 0 + local proxy=haproxy + local cr_yaml=/tmp/tmp.Zvl3zkw7nj/cr_1.19.0_haproxy.yaml + prepare_cr_yaml /tmp/tmp.Zvl3zkw7nj/cr_1.19.0_haproxy.yaml haproxy upgrade-haproxy 3 v1.19.0 + local cr_yaml=/tmp/tmp.Zvl3zkw7nj/cr_1.19.0_haproxy.yaml + local proxy=haproxy + local cluster=upgrade-haproxy + local cluster_size=3 + local git_tag=v1.19.0 + yq eval ' .metadata.name = "upgrade-haproxy" | .spec.secretsName = "my-cluster-secrets" | .spec.vaultSecretName = "some-name-vault" | .spec.sslSecretName = "some-name-ssl" | .spec.sslInternalSecretName = "some-name-ssl-internal" | .spec.upgradeOptions.apply = "disabled" | .spec.pxc.size = 3 | .spec.proxysql.size = 3 | .spec.haproxy.size = 3 | .spec.pxc.image = "-pxc" | .spec.proxysql.image = "-proxysql" | .spec.haproxy.image = "-haproxy" | .spec.backup.image = "-backup" | .spec.backup.storages.minio.s3.credentialsSecret = "minio-secret" | .spec.backup.storages.minio.s3.region = "us-east-1" | .spec.backup.storages.minio.s3.bucket = "operator-testing" | .spec.backup.storages.minio.s3.endpointUrl = "http://minio-service.#namespace:9000/" | .spec.backup.storages.minio.type = "s3" | .spec.pmm.image = "-pmm" ' - + curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.19.0/deploy/cr.yaml + [[ haproxy == \h\a\p\r\o\x\y ]] + yq -i eval ' .spec.haproxy.enabled = true | .spec.proxysql.enabled = false ' /tmp/tmp.Zvl3zkw7nj/cr_1.19.0_haproxy.yaml + [[ -n '' ]] + spinup_pxc upgrade-haproxy /tmp/tmp.Zvl3zkw7nj/cr_1.19.0_haproxy.yaml 3 30 /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/secrets_without_tls.yml + local cluster=upgrade-haproxy + local config=/tmp/tmp.Zvl3zkw7nj/cr_1.19.0_haproxy.yaml + local size=3 + local sleep=30 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/secrets_without_tls.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/secrets_without_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.CgESidyyjc ++ mktemp + local LAST_ERR=/tmp/tmp.PzNoj4ER3F + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/secrets_without_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CgESidyyjc secret/my-cluster-secrets created + cat /tmp/tmp.PzNoj4ER3F + rm /tmp/tmp.CgESidyyjc /tmp/tmp.PzNoj4ER3F + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-logcollector$#image: percona/fluentbit:4.0.1-1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: percona/haproxy:2.8.17#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: percona/proxysql2:2.7.3-1.2#' + /usr/bin/sed -e 's#image:.*-backup$#image: percona/percona-xtrabackup:2.4.29#' + /usr/bin/sed -e 's#image:.*-pmm$#image: percona/pmm-client:3.5.0#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:5.7.44-31.65#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:5.7.44-31.65#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:1.19.0#' + kubectl_bin apply -f - + /usr/bin/sed -e s~minio-service.#namespace~minio-service.upgrade-haproxy-28003~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + local LAST_OUT=/tmp/tmp.th0loiCtzE ++ mktemp + local LAST_ERR=/tmp/tmp.tJRf57RlRo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.th0loiCtzE deployment.apps/pxc-client created + cat /tmp/tmp.tJRf57RlRo + rm /tmp/tmp.th0loiCtzE /tmp/tmp.tJRf57RlRo + return 0 + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ upgrade-haproxy == \d\e\m\a\n\d\-\b\a\c\k\u\p ]] + [[ upgrade-haproxy == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]] + apply_config /tmp/tmp.Zvl3zkw7nj/cr_1.19.0_haproxy.yaml + local config_file=/tmp/tmp.Zvl3zkw7nj/cr_1.19.0_haproxy.yaml + local pvc_name= + '[' -z '' ']' + cat_config /tmp/tmp.Zvl3zkw7nj/cr_1.19.0_haproxy.yaml '' + local input_file=/tmp/tmp.Zvl3zkw7nj/cr_1.19.0_haproxy.yaml + local pvc_name= + /usr/bin/sed -e 's#image:.*-logcollector$#image: percona/fluentbit:4.0.1-1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: percona/haproxy:2.8.17#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: percona/proxysql2:2.7.3-1.2#' + /usr/bin/sed -e 's#image:.*-backup$#image: percona/percona-xtrabackup:2.4.29#' + /usr/bin/sed -e 's#image:.*-pmm$#image: percona/pmm-client:3.5.0#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + cat /tmp/tmp.Zvl3zkw7nj/cr_1.19.0_haproxy.yaml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:5.7.44-31.65#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:5.7.44-31.65#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:1.19.0#' + kubectl_bin apply -f - + /usr/bin/sed -e s~minio-service.#namespace~minio-service.upgrade-haproxy-28003~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + local LAST_OUT=/tmp/tmp.R5nLqwwLCm ++ mktemp + local LAST_ERR=/tmp/tmp.U73ubsAVyl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.R5nLqwwLCm perconaxtradbcluster.pxc.percona.com/upgrade-haproxy created + cat /tmp/tmp.U73ubsAVyl + rm /tmp/tmp.R5nLqwwLCm /tmp/tmp.U73ubsAVyl + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy upgrade-haproxy ++ local target_cluster=upgrade-haproxy +++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.M93jnLfP9u ++++ mktemp +++ local LAST_ERR=/tmp/tmp.htzPORH0RC +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.M93jnLfP9u +++ cat /tmp/tmp.htzPORH0RC +++ rm /tmp/tmp.M93jnLfP9u /tmp/tmp.htzPORH0RC +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo upgrade-haproxy-haproxy ++ return + local proxy=upgrade-haproxy-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n upgrade-haproxy-28003 ++ mktemp + local LAST_OUT=/tmp/tmp.fF63uGnYN4 ++ mktemp + local LAST_ERR=/tmp/tmp.0Rg4yu1TML + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n upgrade-haproxy-28003 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n upgrade-haproxy-28003 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n upgrade-haproxy-28003 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.fF63uGnYN4 + cat /tmp/tmp.0Rg4yu1TML error: no matching resources found + rm /tmp/tmp.fF63uGnYN4 /tmp/tmp.0Rg4yu1TML + return 1 + true + wait_for_running upgrade-haproxy-haproxy 1 + local name=upgrade-haproxy-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-haproxy-0 480 + local pod=upgrade-haproxy-haproxy-0 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/upgrade-haproxy-haproxy-0 condition met waiting for pod/upgrade-haproxy-haproxy-0 to become Ready.Ok + wait_for_running upgrade-haproxy-pxc 3 + local name=upgrade-haproxy-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-0 480 + local pod=upgrade-haproxy-pxc-0 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/upgrade-haproxy-pxc-0 condition met waiting for pod/upgrade-haproxy-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-1 480 + local pod=upgrade-haproxy-pxc-1 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/upgrade-haproxy-pxc-1 condition met waiting for pod/upgrade-haproxy-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-2 480 + local pod=upgrade-haproxy-pxc-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo upgrade-haproxy-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/upgrade-haproxy-pxc-2 condition met waiting for pod/upgrade-haproxy-pxc-2 to become Ready.Ok + sleep 30 ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.7H1EBvt1ty +++ mktemp ++ local LAST_ERR=/tmp/tmp.kWHkPQcDkD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7H1EBvt1ty ++ cat /tmp/tmp.kWHkPQcDkD ++ rm /tmp/tmp.7H1EBvt1ty /tmp/tmp.kWHkPQcDkD ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] ++ is_keyring_plugin_in_use upgrade-haproxy ++ local cluster=upgrade-haproxy ++ [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] ++ grep -E -o 'early-plugin-load=keyring_\w+.so' ++ kubectl exec upgrade-haproxy-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ return 1 + [[ -n '' ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h upgrade-haproxy-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h upgrade-haproxy-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.njv5pqqtJe +++ mktemp ++ local LAST_ERR=/tmp/tmp.70e97UiU2p ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.njv5pqqtJe ++ cat /tmp/tmp.70e97UiU2p ++ rm /tmp/tmp.njv5pqqtJe /tmp/tmp.70e97UiU2p ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ echo pxc-client-6c975684f9-4j5ht ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h upgrade-haproxy-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h upgrade-haproxy-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1dUpejEUiL +++ mktemp ++ local LAST_ERR=/tmp/tmp.RsyPLVQ941 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1dUpejEUiL ++ cat /tmp/tmp.RsyPLVQ941 ++ rm /tmp/tmp.1dUpejEUiL /tmp/tmp.RsyPLVQ941 ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ echo pxc-client-6c975684f9-4j5ht ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EjfSmez4a9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.uADDIGzZ3d ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EjfSmez4a9 ++ cat /tmp/tmp.uADDIGzZ3d ++ rm /tmp/tmp.EjfSmez4a9 /tmp/tmp.uADDIGzZ3d ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6c975684f9-4j5ht ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Zvl3zkw7nj/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Zvl3zkw7nj/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.Zvl3zkw7nj/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Fsjv1NSF7a +++ mktemp ++ local LAST_ERR=/tmp/tmp.IL301IwaVv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Fsjv1NSF7a ++ cat /tmp/tmp.IL301IwaVv ++ rm /tmp/tmp.Fsjv1NSF7a /tmp/tmp.IL301IwaVv ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6c975684f9-4j5ht ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Zvl3zkw7nj/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Zvl3zkw7nj/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.Zvl3zkw7nj/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ddzidEBRN6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4a5gJQYvIQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ddzidEBRN6 ++ cat /tmp/tmp.4a5gJQYvIQ ++ rm /tmp/tmp.ddzidEBRN6 /tmp/tmp.4a5gJQYvIQ ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6c975684f9-4j5ht + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Zvl3zkw7nj/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Zvl3zkw7nj/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.Zvl3zkw7nj/select-1.sql + is_keyring_plugin_in_use upgrade-haproxy + local cluster=upgrade-haproxy + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + grep -E -o 'early-plugin-load=keyring_\w+.so' + kubectl exec upgrade-haproxy-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + return 1 + compare_generation 1 haproxy upgrade-haproxy + local generation=1 + local proxy=haproxy + local cluster=upgrade-haproxy + local current_generation + [[ haproxy == \h\a\p\r\o\x\y ]] + containers=(pxc haproxy) + for container in '"${containers[@]}"' + check_generation 1 pxc upgrade-haproxy + local generation=1 + local container=pxc + local cluster=upgrade-haproxy + local current_generation ++ kubectl_bin get statefulset upgrade-haproxy-pxc -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VU4q6RHXwv +++ mktemp ++ local LAST_ERR=/tmp/tmp.3q20QjLF4k ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get statefulset upgrade-haproxy-pxc -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VU4q6RHXwv ++ cat /tmp/tmp.3q20QjLF4k ++ rm /tmp/tmp.VU4q6RHXwv /tmp/tmp.3q20QjLF4k ++ return 0 + current_generation=1 + [[ 1 != \1 ]] + for container in '"${containers[@]}"' + check_generation 1 haproxy upgrade-haproxy + local generation=1 + local container=haproxy + local cluster=upgrade-haproxy + local current_generation ++ kubectl_bin get statefulset upgrade-haproxy-haproxy -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GdZQ77DHww +++ mktemp ++ local LAST_ERR=/tmp/tmp.0ry9Y86cBe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get statefulset upgrade-haproxy-haproxy -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GdZQ77DHww ++ cat /tmp/tmp.0ry9Y86cBe ++ rm /tmp/tmp.GdZQ77DHww /tmp/tmp.0ry9Y86cBe ++ return 0 + current_generation=1 + [[ 1 != \1 ]] + run_backup upgrade-haproxy on-demand-backup-minio + local cluster=upgrade-haproxy + local backup=on-demand-backup-minio + log 'run pxc-backup/on-demand-backup-minio' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-04-08T18:56:17+0000]' run pxc-backup/on-demand-backup-minio [2026-04-08T18:56:17+0000] run pxc-backup/on-demand-backup-minio + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/conf/on-demand-backup-minio.yml ++ mktemp + local LAST_OUT=/tmp/tmp.0HkatrJWW8 ++ mktemp + local LAST_ERR=/tmp/tmp.YY99NWSGxf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/conf/on-demand-backup-minio.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0HkatrJWW8 perconaxtradbclusterbackup.pxc.percona.com/on-demand-backup-minio created + cat /tmp/tmp.YY99NWSGxf + rm /tmp/tmp.0HkatrJWW8 /tmp/tmp.YY99NWSGxf + return 0 + wait_backup on-demand-backup-minio + local backup=on-demand-backup-minio + local status=Succeeded + set +o xtrace waiting for pxc-backup/on-demand-backup-minio to reach Succeeded state......................................Succeeded + desc 'upgrade operator' + set +o xtrace ----------------------------------------------------------------------------------- upgrade operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.1qHmbTI6pG ++ mktemp + local LAST_ERR=/tmp/tmp.pkYVK4LpDn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1qHmbTI6pG customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.pkYVK4LpDn + rm /tmp/tmp.1qHmbTI6pG /tmp/tmp.pkYVK4LpDn + return 0 + [[ -n pxc-operator ]] + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AOD7qbm8vI ++ mktemp + local LAST_ERR=/tmp/tmp.TrCJpiihpU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AOD7qbm8vI clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.TrCJpiihpU + rm /tmp/tmp.AOD7qbm8vI /tmp/tmp.TrCJpiihpU + return 0 + kubectl_bin patch deployment percona-xtradb-cluster-operator '-p{"spec":{"template":{"spec":{"containers":[{"name":"percona-xtradb-cluster-operator","image":"perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd"}]}}}}' -n pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ik0oLPFrvF ++ mktemp + local LAST_ERR=/tmp/tmp.AxMkf9hcHk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch deployment percona-xtradb-cluster-operator '-p{"spec":{"template":{"spec":{"containers":[{"name":"percona-xtradb-cluster-operator","image":"perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd"}]}}}}' -n pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ik0oLPFrvF deployment.apps/percona-xtradb-cluster-operator patched + cat /tmp/tmp.AxMkf9hcHk + rm /tmp/tmp.ik0oLPFrvF /tmp/tmp.AxMkf9hcHk + return 0 + kubectl_bin rollout status deployment/percona-xtradb-cluster-operator -n pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.S5weHq08Fd ++ mktemp + local LAST_ERR=/tmp/tmp.6tYto4LOJr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl rollout status deployment/percona-xtradb-cluster-operator -n pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.S5weHq08Fd Waiting for deployment "percona-xtradb-cluster-operator" rollout to finish: 0 of 1 updated replicas are available... deployment "percona-xtradb-cluster-operator" successfully rolled out + cat /tmp/tmp.6tYto4LOJr + rm /tmp/tmp.S5weHq08Fd /tmp/tmp.6tYto4LOJr + return 0 + sleep 10 + desc 'wait for operator upgrade' + set +o xtrace ----------------------------------------------------------------------------------- wait for operator upgrade ----------------------------------------------------------------------------------- + local i=0 + local max=60 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image' -n pxc-operator ++ awk '{print $1}' ++ grep -vc NAME +++ mktemp ++ local LAST_OUT=/tmp/tmp.rnHcVGnGkE +++ mktemp ++ local LAST_ERR=/tmp/tmp.fr08qVl8Vx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rnHcVGnGkE ++ cat /tmp/tmp.fr08qVl8Vx ++ rm /tmp/tmp.rnHcVGnGkE /tmp/tmp.fr08qVl8Vx ++ return 0 + [[ 1 -eq 1 ]] + '[' -n pxc-operator ']' ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.8Y1h6IzhbX +++ mktemp ++ local LAST_ERR=/tmp/tmp.bWEaaSrKFu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8Y1h6IzhbX ++ cat /tmp/tmp.bWEaaSrKFu ++ rm /tmp/tmp.8Y1h6IzhbX /tmp/tmp.bWEaaSrKFu ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster3 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.mKLlt8lbEY ++ mktemp + local LAST_ERR=/tmp/tmp.QAcZA8oYG7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster3 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mKLlt8lbEY Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster3" modified. + cat /tmp/tmp.QAcZA8oYG7 + rm /tmp/tmp.mKLlt8lbEY /tmp/tmp.QAcZA8oYG7 + return 0 ++ grep perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd ++ awk '{print $1}' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1YOHdsIuqj +++ mktemp ++ local LAST_ERR=/tmp/tmp.dtUeEy3tQP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1YOHdsIuqj ++ cat /tmp/tmp.dtUeEy3tQP ++ rm /tmp/tmp.1YOHdsIuqj /tmp/tmp.dtUeEy3tQP ++ return 0 + wait_pod percona-xtradb-cluster-operator-6cfdc6cb68-cq7vh + local pod=percona-xtradb-cluster-operator-6cfdc6cb68-cq7vh + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo percona-xtradb-cluster-operator-6cfdc6cb68-cq7vh ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-6cfdc6cb68-cq7vh condition met waiting for pod/percona-xtradb-cluster-operator-6cfdc6cb68-cq7vh to become Ready.Ok ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.CXdwG5Sczd +++ mktemp ++ local LAST_ERR=/tmp/tmp.bFbkH1IvSJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CXdwG5Sczd ++ cat /tmp/tmp.bFbkH1IvSJ ++ rm /tmp/tmp.CXdwG5Sczd /tmp/tmp.bFbkH1IvSJ ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster3 --namespace=upgrade-haproxy-28003 ++ mktemp + local LAST_OUT=/tmp/tmp.JeZxMOKjJx ++ mktemp + local LAST_ERR=/tmp/tmp.NzdNI1MA8J + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster3 --namespace=upgrade-haproxy-28003 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JeZxMOKjJx Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2428-b77708dd-1-cluster3" modified. + cat /tmp/tmp.NzdNI1MA8J + rm /tmp/tmp.JeZxMOKjJx /tmp/tmp.NzdNI1MA8J + return 0 + desc 'check images and generation after operator upgrade' + set +o xtrace ----------------------------------------------------------------------------------- check images and generation after operator upgrade ----------------------------------------------------------------------------------- + check_pxc_liveness upgrade-haproxy 3 + local cluster=upgrade-haproxy + local cluster_size=3 + wait_cluster_consistency upgrade-haproxy 3 + local cluster_name=upgrade-haproxy + local cluster_size=3 + local proxy_size= + '[' -z '' ']' ++ get_proxy_size upgrade-haproxy ++ local cluster=upgrade-haproxy +++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.A5rRoYsNN8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.deZgHH7UxG +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.A5rRoYsNN8 +++ cat /tmp/tmp.deZgHH7UxG +++ rm /tmp/tmp.A5rRoYsNN8 /tmp/tmp.deZgHH7UxG +++ return 0 ++ [[ true == \t\r\u\e ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.size}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1dQMa45Rb2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.J0LjwY8Jtj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.size}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1dQMa45Rb2 ++ cat /tmp/tmp.J0LjwY8Jtj ++ rm /tmp/tmp.1dQMa45Rb2 /tmp/tmp.J0LjwY8Jtj ++ return 0 ++ return + proxy_size=3 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=300 + sleep 7 + echo -n 'waiting for pxc/upgrade-haproxy to be ready' waiting for pxc/upgrade-haproxy to be ready++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pxs3wvpK8K +++ mktemp ++ local LAST_ERR=/tmp/tmp.HJJAzcpYgM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Pxs3wvpK8K ++ cat /tmp/tmp.HJJAzcpYgM ++ rm /tmp/tmp.Pxs3wvpK8K /tmp/tmp.HJJAzcpYgM ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ck04j5pSdN +++ mktemp ++ local LAST_ERR=/tmp/tmp.8W91SNLMtx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ck04j5pSdN ++ cat /tmp/tmp.8W91SNLMtx ++ rm /tmp/tmp.ck04j5pSdN /tmp/tmp.8W91SNLMtx ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine upgrade-haproxy +++ local cluster_name=upgrade-haproxy ++++ get_proxy upgrade-haproxy ++++ local target_cluster=upgrade-haproxy +++++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.tSAWCQ3OLG ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Akmp95oQcb +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.tSAWCQ3OLG +++++ cat /tmp/tmp.Akmp95oQcb +++++ rm /tmp/tmp.tSAWCQ3OLG /tmp/tmp.Akmp95oQcb +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo upgrade-haproxy-haproxy ++++ return +++ local cluster_proxy=upgrade-haproxy-haproxy +++ echo haproxy ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YBHUL6d27r +++ mktemp ++ local LAST_ERR=/tmp/tmp.VHuPe1D9Kn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YBHUL6d27r ++ cat /tmp/tmp.VHuPe1D9Kn ++ rm /tmp/tmp.YBHUL6d27r /tmp/tmp.VHuPe1D9Kn ++ return 0 + [[ 3 == \3 ]] + echo + wait_for_running upgrade-haproxy-pxc 3 + local name=upgrade-haproxy-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-0 480 + local pod=upgrade-haproxy-pxc-0 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/upgrade-haproxy-pxc-0 condition met waiting for pod/upgrade-haproxy-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-1 480 + local pod=upgrade-haproxy-pxc-1 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-pxc-1 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/upgrade-haproxy-pxc-1 condition met waiting for pod/upgrade-haproxy-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-2 480 + local pod=upgrade-haproxy-pxc-2 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/upgrade-haproxy-pxc-2 condition met waiting for pod/upgrade-haproxy-pxc-2 to become Ready.Ok ++ seq 0 2 + for i in '$(seq 0 $((cluster_size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7zxJVCfz61 +++ mktemp ++ local LAST_ERR=/tmp/tmp.h8zobCp52s ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7zxJVCfz61 ++ cat /tmp/tmp.h8zobCp52s ++ rm /tmp/tmp.7zxJVCfz61 /tmp/tmp.h8zobCp52s ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ echo pxc-client-6c975684f9-4j5ht ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Zvl3zkw7nj/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Zvl3zkw7nj/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.Zvl3zkw7nj/select-1.sql + for i in '$(seq 0 $((cluster_size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tCcY0ze5Fj +++ mktemp ++ local LAST_ERR=/tmp/tmp.gqd4UvZUBD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tCcY0ze5Fj ++ cat /tmp/tmp.gqd4UvZUBD ++ rm /tmp/tmp.tCcY0ze5Fj /tmp/tmp.gqd4UvZUBD ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6c975684f9-4j5ht ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Zvl3zkw7nj/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Zvl3zkw7nj/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.Zvl3zkw7nj/select-1.sql + for i in '$(seq 0 $((cluster_size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JUb1SkJvOP +++ mktemp ++ local LAST_ERR=/tmp/tmp.vrHvximYMl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JUb1SkJvOP ++ cat /tmp/tmp.vrHvximYMl ++ rm /tmp/tmp.JUb1SkJvOP /tmp/tmp.vrHvximYMl ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ echo pxc-client-6c975684f9-4j5ht ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Zvl3zkw7nj/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Zvl3zkw7nj/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.Zvl3zkw7nj/select-1.sql ++ kubectl_bin get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gjj5a9xwen +++ mktemp ++ local LAST_ERR=/tmp/tmp.9JVXrtII4z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gjj5a9xwen ++ cat /tmp/tmp.9JVXrtII4z ++ rm /tmp/tmp.gjj5a9xwen /tmp/tmp.9JVXrtII4z ++ return 0 + [[ perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd == perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.proxysql.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bvroiqqba4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.LMEikSj4y8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.proxysql.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bvroiqqba4 ++ cat /tmp/tmp.LMEikSj4y8 ++ rm /tmp/tmp.bvroiqqba4 /tmp/tmp.LMEikSj4y8 ++ return 0 + [[ percona/proxysql2:2.7.3-1.2 == percona/proxysql2:2.7.3-1.2 ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RZTVt7DCbo +++ mktemp ++ local LAST_ERR=/tmp/tmp.tNwLX1a0Qv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RZTVt7DCbo ++ cat /tmp/tmp.tNwLX1a0Qv ++ rm /tmp/tmp.RZTVt7DCbo /tmp/tmp.tNwLX1a0Qv ++ return 0 + [[ percona/haproxy:2.8.17 == percona/haproxy:2.8.17 ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.backup.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XtLuwtE6td +++ mktemp ++ local LAST_ERR=/tmp/tmp.hWdwPeLeJf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.backup.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XtLuwtE6td ++ cat /tmp/tmp.hWdwPeLeJf ++ rm /tmp/tmp.XtLuwtE6td /tmp/tmp.hWdwPeLeJf ++ return 0 + [[ percona/percona-xtrabackup:2.4.29 == percona/percona-xtrabackup:2.4.29 ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.pmm.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gesrPrjs8l +++ mktemp ++ local LAST_ERR=/tmp/tmp.06mhS8Hsdp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.pmm.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gesrPrjs8l ++ cat /tmp/tmp.06mhS8Hsdp ++ rm /tmp/tmp.gesrPrjs8l /tmp/tmp.06mhS8Hsdp ++ return 0 + [[ percona/pmm-client:3.5.0 == percona/pmm-client:3.5.0 ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.logcollector.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pjV2nZM3U5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.kopg6erPz2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.logcollector.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pjV2nZM3U5 ++ cat /tmp/tmp.kopg6erPz2 ++ rm /tmp/tmp.pjV2nZM3U5 /tmp/tmp.kopg6erPz2 ++ return 0 + [[ percona/fluentbit:4.0.1-1 == percona/fluentbit:4.0.1-1 ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.pxc.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.u6gcDyfaZj +++ mktemp ++ local LAST_ERR=/tmp/tmp.HzE7Gd23Kq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.pxc.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u6gcDyfaZj ++ cat /tmp/tmp.HzE7Gd23Kq ++ rm /tmp/tmp.u6gcDyfaZj /tmp/tmp.HzE7Gd23Kq ++ return 0 + [[ percona/percona-xtradb-cluster:5.7.44-31.65 == percona/percona-xtradb-cluster:5.7.44-31.65 ]] + : Operator image has been updated correctly + compare_generation 1 haproxy upgrade-haproxy + local generation=1 + local proxy=haproxy + local cluster=upgrade-haproxy + local current_generation + [[ haproxy == \h\a\p\r\o\x\y ]] + containers=(pxc haproxy) + for container in '"${containers[@]}"' + check_generation 1 pxc upgrade-haproxy + local generation=1 + local container=pxc + local cluster=upgrade-haproxy + local current_generation ++ kubectl_bin get statefulset upgrade-haproxy-pxc -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LTBh8RixBT +++ mktemp ++ local LAST_ERR=/tmp/tmp.sm8uUisANg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get statefulset upgrade-haproxy-pxc -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LTBh8RixBT ++ cat /tmp/tmp.sm8uUisANg ++ rm /tmp/tmp.LTBh8RixBT /tmp/tmp.sm8uUisANg ++ return 0 + current_generation=1 + [[ 1 != \1 ]] + for container in '"${containers[@]}"' + check_generation 1 haproxy upgrade-haproxy + local generation=1 + local container=haproxy + local cluster=upgrade-haproxy + local current_generation ++ kubectl_bin get statefulset upgrade-haproxy-haproxy -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.G45s8wv1x5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.AZ1vM9en9e ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get statefulset upgrade-haproxy-haproxy -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.G45s8wv1x5 ++ cat /tmp/tmp.AZ1vM9en9e ++ rm /tmp/tmp.G45s8wv1x5 /tmp/tmp.AZ1vM9en9e ++ return 0 + current_generation=1 + [[ 1 != \1 ]] + desc 'patch pxc images and upgrade' + set +o xtrace ----------------------------------------------------------------------------------- patch pxc images and upgrade ----------------------------------------------------------------------------------- + kubectl_bin patch pxc upgrade-haproxy --type=merge --patch '{ "spec": { "crVersion": "1.20.0", "pxc": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc5.7" }, "pmm": { "image": "perconalab/pmm-client:dev-latest" }, "haproxy": { "image": "perconalab/percona-xtradb-cluster-operator:main-haproxy" }, "proxysql": { "image": "perconalab/percona-xtradb-cluster-operator:main-proxysql" }, "logcollector": { "image": "perconalab/fluentbit:main-logcollector" }, "backup": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup" } }}' ++ mktemp + local LAST_OUT=/tmp/tmp.UUO5d9Op22 ++ mktemp + local LAST_ERR=/tmp/tmp.ywiAjPesxf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc upgrade-haproxy --type=merge --patch '{ "spec": { "crVersion": "1.20.0", "pxc": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc5.7" }, "pmm": { "image": "perconalab/pmm-client:dev-latest" }, "haproxy": { "image": "perconalab/percona-xtradb-cluster-operator:main-haproxy" }, "proxysql": { "image": "perconalab/percona-xtradb-cluster-operator:main-proxysql" }, "logcollector": { "image": "perconalab/fluentbit:main-logcollector" }, "backup": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup" } }}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UUO5d9Op22 perconaxtradbcluster.pxc.percona.com/upgrade-haproxy patched + cat /tmp/tmp.ywiAjPesxf + rm /tmp/tmp.UUO5d9Op22 /tmp/tmp.ywiAjPesxf + return 0 + sleep 10 + desc 'check images and generation after full upgrade' + set +o xtrace ----------------------------------------------------------------------------------- check images and generation after full upgrade ----------------------------------------------------------------------------------- + check_pxc_liveness upgrade-haproxy 3 + local cluster=upgrade-haproxy + local cluster_size=3 + wait_cluster_consistency upgrade-haproxy 3 + local cluster_name=upgrade-haproxy + local cluster_size=3 + local proxy_size= + '[' -z '' ']' ++ get_proxy_size upgrade-haproxy ++ local cluster=upgrade-haproxy +++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XG8YfkGril ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GPGh3fmeCL +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.XG8YfkGril +++ cat /tmp/tmp.GPGh3fmeCL +++ rm /tmp/tmp.XG8YfkGril /tmp/tmp.GPGh3fmeCL +++ return 0 ++ [[ true == \t\r\u\e ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.size}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KY1xlsJtmE +++ mktemp ++ local LAST_ERR=/tmp/tmp.m6EGOAJVYC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.size}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KY1xlsJtmE ++ cat /tmp/tmp.m6EGOAJVYC ++ rm /tmp/tmp.KY1xlsJtmE /tmp/tmp.m6EGOAJVYC ++ return 0 ++ return + proxy_size=3 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=300 + sleep 7 + echo -n 'waiting for pxc/upgrade-haproxy to be ready' waiting for pxc/upgrade-haproxy to be ready++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WHom04frIp +++ mktemp ++ local LAST_ERR=/tmp/tmp.GD00XUmlI2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WHom04frIp ++ cat /tmp/tmp.GD00XUmlI2 ++ rm /tmp/tmp.WHom04frIp /tmp/tmp.GD00XUmlI2 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 0 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.M6x9ra2l3g +++ mktemp ++ local LAST_ERR=/tmp/tmp.O9ck3Cc8rg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.M6x9ra2l3g ++ cat /tmp/tmp.O9ck3Cc8rg ++ rm /tmp/tmp.M6x9ra2l3g /tmp/tmp.O9ck3Cc8rg ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 1 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CyIVvJPaA7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Kjp2Hnmhp2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CyIVvJPaA7 ++ cat /tmp/tmp.Kjp2Hnmhp2 ++ rm /tmp/tmp.CyIVvJPaA7 /tmp/tmp.Kjp2Hnmhp2 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 2 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CYQplAEim9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.EsEUMGdSTq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CYQplAEim9 ++ cat /tmp/tmp.EsEUMGdSTq ++ rm /tmp/tmp.CYQplAEim9 /tmp/tmp.EsEUMGdSTq ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 3 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9sDGUulzz2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gwGOh6CM32 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9sDGUulzz2 ++ cat /tmp/tmp.gwGOh6CM32 ++ rm /tmp/tmp.9sDGUulzz2 /tmp/tmp.gwGOh6CM32 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 4 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7rD6kkYkMA +++ mktemp ++ local LAST_ERR=/tmp/tmp.YlmUhkyQlL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7rD6kkYkMA ++ cat /tmp/tmp.YlmUhkyQlL ++ rm /tmp/tmp.7rD6kkYkMA /tmp/tmp.YlmUhkyQlL ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 5 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZNsz2USts7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.aCN4na4n6n ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZNsz2USts7 ++ cat /tmp/tmp.aCN4na4n6n ++ rm /tmp/tmp.ZNsz2USts7 /tmp/tmp.aCN4na4n6n ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 6 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Igu7hnQ8Uc +++ mktemp ++ local LAST_ERR=/tmp/tmp.TrIuDM5yvp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Igu7hnQ8Uc ++ cat /tmp/tmp.TrIuDM5yvp ++ rm /tmp/tmp.Igu7hnQ8Uc /tmp/tmp.TrIuDM5yvp ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 7 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5Rqrjk9Jpx +++ mktemp ++ local LAST_ERR=/tmp/tmp.g3k4C1Zgyh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5Rqrjk9Jpx ++ cat /tmp/tmp.g3k4C1Zgyh ++ rm /tmp/tmp.5Rqrjk9Jpx /tmp/tmp.g3k4C1Zgyh ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 8 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.L32Z7PBPql +++ mktemp ++ local LAST_ERR=/tmp/tmp.SD98Ex2w1Y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.L32Z7PBPql ++ cat /tmp/tmp.SD98Ex2w1Y ++ rm /tmp/tmp.L32Z7PBPql /tmp/tmp.SD98Ex2w1Y ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 9 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QdZDb8a6I7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.chjWjSc4eu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QdZDb8a6I7 ++ cat /tmp/tmp.chjWjSc4eu ++ rm /tmp/tmp.QdZDb8a6I7 /tmp/tmp.chjWjSc4eu ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 10 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cln6eD0xS9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Fk6TAaOYiL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cln6eD0xS9 ++ cat /tmp/tmp.Fk6TAaOYiL ++ rm /tmp/tmp.cln6eD0xS9 /tmp/tmp.Fk6TAaOYiL ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 11 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dyKqjSPBCj +++ mktemp ++ local LAST_ERR=/tmp/tmp.NRrqsKNakf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dyKqjSPBCj ++ cat /tmp/tmp.NRrqsKNakf ++ rm /tmp/tmp.dyKqjSPBCj /tmp/tmp.NRrqsKNakf ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 12 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZZW0NsOrtt +++ mktemp ++ local LAST_ERR=/tmp/tmp.yCpwPJIK67 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZZW0NsOrtt ++ cat /tmp/tmp.yCpwPJIK67 ++ rm /tmp/tmp.ZZW0NsOrtt /tmp/tmp.yCpwPJIK67 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 13 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oJehsio7EX +++ mktemp ++ local LAST_ERR=/tmp/tmp.GL9E6zdFPr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oJehsio7EX ++ cat /tmp/tmp.GL9E6zdFPr ++ rm /tmp/tmp.oJehsio7EX /tmp/tmp.GL9E6zdFPr ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 14 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ngrts18t7g +++ mktemp ++ local LAST_ERR=/tmp/tmp.Zd2IXuJFXM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ngrts18t7g ++ cat /tmp/tmp.Zd2IXuJFXM ++ rm /tmp/tmp.ngrts18t7g /tmp/tmp.Zd2IXuJFXM ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 15 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MwD40mSJNq +++ mktemp ++ local LAST_ERR=/tmp/tmp.EsLPvnFOdN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MwD40mSJNq ++ cat /tmp/tmp.EsLPvnFOdN ++ rm /tmp/tmp.MwD40mSJNq /tmp/tmp.EsLPvnFOdN ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 16 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wdaNvG2MXI +++ mktemp ++ local LAST_ERR=/tmp/tmp.aPxLMb2tVH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wdaNvG2MXI ++ cat /tmp/tmp.aPxLMb2tVH ++ rm /tmp/tmp.wdaNvG2MXI /tmp/tmp.aPxLMb2tVH ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 17 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZQax2W0wrp +++ mktemp ++ local LAST_ERR=/tmp/tmp.P7u1RFDwFS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZQax2W0wrp ++ cat /tmp/tmp.P7u1RFDwFS ++ rm /tmp/tmp.ZQax2W0wrp /tmp/tmp.P7u1RFDwFS ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 18 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CBx2SDlvMP +++ mktemp ++ local LAST_ERR=/tmp/tmp.kDWWyCQ6AQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CBx2SDlvMP ++ cat /tmp/tmp.kDWWyCQ6AQ ++ rm /tmp/tmp.CBx2SDlvMP /tmp/tmp.kDWWyCQ6AQ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 19 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.u8mI93zk0m +++ mktemp ++ local LAST_ERR=/tmp/tmp.IYkw1o8sH0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u8mI93zk0m ++ cat /tmp/tmp.IYkw1o8sH0 ++ rm /tmp/tmp.u8mI93zk0m /tmp/tmp.IYkw1o8sH0 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 20 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DywPIA2Bcs +++ mktemp ++ local LAST_ERR=/tmp/tmp.eRRTMmYuCW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DywPIA2Bcs ++ cat /tmp/tmp.eRRTMmYuCW ++ rm /tmp/tmp.DywPIA2Bcs /tmp/tmp.eRRTMmYuCW ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 21 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.h3x5VFSLvO +++ mktemp ++ local LAST_ERR=/tmp/tmp.uQ9S4ttxo5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.h3x5VFSLvO ++ cat /tmp/tmp.uQ9S4ttxo5 ++ rm /tmp/tmp.h3x5VFSLvO /tmp/tmp.uQ9S4ttxo5 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 22 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QhdJJXd3lo +++ mktemp ++ local LAST_ERR=/tmp/tmp.LubqG5J0DX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QhdJJXd3lo ++ cat /tmp/tmp.LubqG5J0DX ++ rm /tmp/tmp.QhdJJXd3lo /tmp/tmp.LubqG5J0DX ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 23 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9U8nXy9sZ6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.M6jfseiLnZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9U8nXy9sZ6 ++ cat /tmp/tmp.M6jfseiLnZ ++ rm /tmp/tmp.9U8nXy9sZ6 /tmp/tmp.M6jfseiLnZ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 24 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GXc7iDvd5r +++ mktemp ++ local LAST_ERR=/tmp/tmp.h4jQcnB1Wy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GXc7iDvd5r ++ cat /tmp/tmp.h4jQcnB1Wy ++ rm /tmp/tmp.GXc7iDvd5r /tmp/tmp.h4jQcnB1Wy ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 25 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KZqeNY8LeS +++ mktemp ++ local LAST_ERR=/tmp/tmp.VFvH2frJyB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KZqeNY8LeS ++ cat /tmp/tmp.VFvH2frJyB ++ rm /tmp/tmp.KZqeNY8LeS /tmp/tmp.VFvH2frJyB ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 26 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eVfPxgaWO9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FCUy2M7kSS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eVfPxgaWO9 ++ cat /tmp/tmp.FCUy2M7kSS ++ rm /tmp/tmp.eVfPxgaWO9 /tmp/tmp.FCUy2M7kSS ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 27 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jdln0kkT4y +++ mktemp ++ local LAST_ERR=/tmp/tmp.46BvRR7tyq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jdln0kkT4y ++ cat /tmp/tmp.46BvRR7tyq ++ rm /tmp/tmp.jdln0kkT4y /tmp/tmp.46BvRR7tyq ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 28 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oz4pKXO9WM +++ mktemp ++ local LAST_ERR=/tmp/tmp.4ftEJGouhL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oz4pKXO9WM ++ cat /tmp/tmp.4ftEJGouhL ++ rm /tmp/tmp.oz4pKXO9WM /tmp/tmp.4ftEJGouhL ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 29 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fVCSUqco8v +++ mktemp ++ local LAST_ERR=/tmp/tmp.AfpKLEKgnl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fVCSUqco8v ++ cat /tmp/tmp.AfpKLEKgnl ++ rm /tmp/tmp.fVCSUqco8v /tmp/tmp.AfpKLEKgnl ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 30 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.P5MWYp9y43 +++ mktemp ++ local LAST_ERR=/tmp/tmp.IlyAfY1aMv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.P5MWYp9y43 ++ cat /tmp/tmp.IlyAfY1aMv ++ rm /tmp/tmp.P5MWYp9y43 /tmp/tmp.IlyAfY1aMv ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 31 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qQ1guRMF51 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Nu9FaXRsp9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qQ1guRMF51 ++ cat /tmp/tmp.Nu9FaXRsp9 ++ rm /tmp/tmp.qQ1guRMF51 /tmp/tmp.Nu9FaXRsp9 ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6z7lgmJb0o +++ mktemp ++ local LAST_ERR=/tmp/tmp.NqDez2HfKT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6z7lgmJb0o ++ cat /tmp/tmp.NqDez2HfKT ++ rm /tmp/tmp.6z7lgmJb0o /tmp/tmp.NqDez2HfKT ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine upgrade-haproxy +++ local cluster_name=upgrade-haproxy ++++ get_proxy upgrade-haproxy ++++ local target_cluster=upgrade-haproxy +++++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.34fV7H2Ujl ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.wK8YEQ3xtC +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.34fV7H2Ujl +++++ cat /tmp/tmp.wK8YEQ3xtC +++++ rm /tmp/tmp.34fV7H2Ujl /tmp/tmp.wK8YEQ3xtC +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo upgrade-haproxy-haproxy ++++ return +++ local cluster_proxy=upgrade-haproxy-haproxy +++ echo haproxy ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.j953rBAewN +++ mktemp ++ local LAST_ERR=/tmp/tmp.OVEFiEnULt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.j953rBAewN ++ cat /tmp/tmp.OVEFiEnULt ++ rm /tmp/tmp.j953rBAewN /tmp/tmp.OVEFiEnULt ++ return 0 + [[ 3 == \3 ]] + echo + wait_for_running upgrade-haproxy-pxc 3 + local name=upgrade-haproxy-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-0 480 + local pod=upgrade-haproxy-pxc-0 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/upgrade-haproxy-pxc-0 condition met waiting for pod/upgrade-haproxy-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-1 480 + local pod=upgrade-haproxy-pxc-1 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/upgrade-haproxy-pxc-1 condition met waiting for pod/upgrade-haproxy-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-2 480 + local pod=upgrade-haproxy-pxc-2 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-pxc-2 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/upgrade-haproxy-pxc-2 condition met waiting for pod/upgrade-haproxy-pxc-2 to become Ready.Ok ++ seq 0 2 + for i in '$(seq 0 $((cluster_size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gx0B5I2wsK +++ mktemp ++ local LAST_ERR=/tmp/tmp.zn8TWVtH1y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gx0B5I2wsK ++ cat /tmp/tmp.zn8TWVtH1y ++ rm /tmp/tmp.gx0B5I2wsK /tmp/tmp.zn8TWVtH1y ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ echo pxc-client-6c975684f9-4j5ht ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Zvl3zkw7nj/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Zvl3zkw7nj/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.Zvl3zkw7nj/select-1.sql + for i in '$(seq 0 $((cluster_size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FkUfLsjRfW +++ mktemp ++ local LAST_ERR=/tmp/tmp.qvxOL5bP1g ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FkUfLsjRfW ++ cat /tmp/tmp.qvxOL5bP1g ++ rm /tmp/tmp.FkUfLsjRfW /tmp/tmp.qvxOL5bP1g ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ echo pxc-client-6c975684f9-4j5ht ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Zvl3zkw7nj/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Zvl3zkw7nj/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.Zvl3zkw7nj/select-1.sql + for i in '$(seq 0 $((cluster_size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9CAlzXSqcX +++ mktemp ++ local LAST_ERR=/tmp/tmp.w6M1nVtNcI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9CAlzXSqcX ++ cat /tmp/tmp.w6M1nVtNcI ++ rm /tmp/tmp.9CAlzXSqcX /tmp/tmp.w6M1nVtNcI ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ echo pxc-client-6c975684f9-4j5ht ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Zvl3zkw7nj/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Zvl3zkw7nj/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.Zvl3zkw7nj/select-1.sql ++ kubectl_bin get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RMrWCOgs7O +++ mktemp ++ local LAST_ERR=/tmp/tmp.2gQJHNMrcA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RMrWCOgs7O ++ cat /tmp/tmp.2gQJHNMrcA ++ rm /tmp/tmp.RMrWCOgs7O /tmp/tmp.2gQJHNMrcA ++ return 0 + [[ perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd == perconalab/percona-xtradb-cluster-operator:PR-2428-b77708dd ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.proxysql.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.H2uKV3cmYF +++ mktemp ++ local LAST_ERR=/tmp/tmp.IS29UEkgCz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.proxysql.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.H2uKV3cmYF ++ cat /tmp/tmp.IS29UEkgCz ++ rm /tmp/tmp.H2uKV3cmYF /tmp/tmp.IS29UEkgCz ++ return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-proxysql == perconalab/percona-xtradb-cluster-operator:main-proxysql ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IlbpjZivSi +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZvSAZYdYMc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IlbpjZivSi ++ cat /tmp/tmp.ZvSAZYdYMc ++ rm /tmp/tmp.IlbpjZivSi /tmp/tmp.ZvSAZYdYMc ++ return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-haproxy == perconalab/percona-xtradb-cluster-operator:main-haproxy ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.backup.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V3J96eomJQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.i2aIsTIgkc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.backup.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.V3J96eomJQ ++ cat /tmp/tmp.i2aIsTIgkc ++ rm /tmp/tmp.V3J96eomJQ /tmp/tmp.i2aIsTIgkc ++ return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup == perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.pmm.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oqj8NQVgTN +++ mktemp ++ local LAST_ERR=/tmp/tmp.0bM2Z5CiEA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.pmm.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oqj8NQVgTN ++ cat /tmp/tmp.0bM2Z5CiEA ++ rm /tmp/tmp.oqj8NQVgTN /tmp/tmp.0bM2Z5CiEA ++ return 0 + [[ perconalab/pmm-client:dev-latest == perconalab/pmm-client:dev-latest ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.logcollector.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pvT5Wbjz5B +++ mktemp ++ local LAST_ERR=/tmp/tmp.7ysrV9bMKi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.logcollector.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pvT5Wbjz5B ++ cat /tmp/tmp.7ysrV9bMKi ++ rm /tmp/tmp.pvT5Wbjz5B /tmp/tmp.7ysrV9bMKi ++ return 0 + [[ perconalab/fluentbit:main-logcollector == perconalab/fluentbit:main-logcollector ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.pxc.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YuHRDkIHVO +++ mktemp ++ local LAST_ERR=/tmp/tmp.vvSRgpBJ7v ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.pxc.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YuHRDkIHVO ++ cat /tmp/tmp.vvSRgpBJ7v ++ rm /tmp/tmp.YuHRDkIHVO /tmp/tmp.vvSRgpBJ7v ++ return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 == perconalab/percona-xtradb-cluster-operator:main-pxc5.7 ]] + : Cluster images have been updated correctly + compare_generation 2 haproxy upgrade-haproxy + local generation=2 + local proxy=haproxy + local cluster=upgrade-haproxy + local current_generation + [[ haproxy == \h\a\p\r\o\x\y ]] + containers=(pxc haproxy) + for container in '"${containers[@]}"' + check_generation 2 pxc upgrade-haproxy + local generation=2 + local container=pxc + local cluster=upgrade-haproxy + local current_generation ++ kubectl_bin get statefulset upgrade-haproxy-pxc -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AvxC44YYzn +++ mktemp ++ local LAST_ERR=/tmp/tmp.4tuCERfyuV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get statefulset upgrade-haproxy-pxc -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AvxC44YYzn ++ cat /tmp/tmp.4tuCERfyuV ++ rm /tmp/tmp.AvxC44YYzn /tmp/tmp.4tuCERfyuV ++ return 0 + current_generation=2 + [[ 2 != \2 ]] + for container in '"${containers[@]}"' + check_generation 2 haproxy upgrade-haproxy + local generation=2 + local container=haproxy + local cluster=upgrade-haproxy + local current_generation ++ kubectl_bin get statefulset upgrade-haproxy-haproxy -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oKku76PY5N +++ mktemp ++ local LAST_ERR=/tmp/tmp.FOpJ5Wiyrt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get statefulset upgrade-haproxy-haproxy -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oKku76PY5N ++ cat /tmp/tmp.FOpJ5Wiyrt ++ rm /tmp/tmp.oKku76PY5N /tmp/tmp.FOpJ5Wiyrt ++ return 0 + current_generation=2 + [[ 2 != \2 ]] + compare_kubectl statefulset/upgrade-haproxy-pxc + local resource=statefulset/upgrade-haproxy-pxc + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-pxc.yml + local new_result=/tmp/tmp.Zvl3zkw7nj/statefulset_upgrade-haproxy-pxc.yml + desc 'compare statefulset/upgrade-haproxy-pxc-' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/upgrade-haproxy-pxc- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-pxc-eks.yml ']' + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + version_gt 1.33 ++ echo '1.32 >= 1.33' ++ bc -l + '[' 0 -eq 1 ']' + return 1 + version_gt 1.29 ++ echo '1.32 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-pxc-k129.yml ']' + version_gt 1.27 ++ echo '1.32 >= 1.27' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-pxc-k127.yml ']' + version_gt 1.24 ++ echo '1.32 >= 1.24' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-pxc-k124.yml ']' + version_gt 1.22 ++ echo '1.32 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-pxc-k122.yml ']' + version_gt 1.21 ++ echo '1.32 >= 1.21' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-pxc-k121.yml ']' + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-pxc-oc.yml ']' + version_gt 1.29 ++ echo '1.32 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-pxc-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-pxc-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-pxc-aks.yml ']' + kubectl_bin get -o yaml statefulset/upgrade-haproxy-pxc ++ mktemp + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("upgrade-haproxy-28003", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + local LAST_OUT=/tmp/tmp.WB0tWG5wr1 ++ mktemp + local LAST_ERR=/tmp/tmp.NibzK5mwO0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/upgrade-haproxy-pxc + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WB0tWG5wr1 + cat /tmp/tmp.NibzK5mwO0 + rm /tmp/tmp.WB0tWG5wr1 /tmp/tmp.NibzK5mwO0 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-pxc.yml /tmp/tmp.Zvl3zkw7nj/statefulset_upgrade-haproxy-pxc.yml + log 'compare_kubectl: statefulset/upgrade-haproxy-pxc OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-04-08T19:03:16+0000]' compare_kubectl: statefulset/upgrade-haproxy-pxc OK [2026-04-08T19:03:16+0000] compare_kubectl: statefulset/upgrade-haproxy-pxc OK + compare_kubectl statefulset/upgrade-haproxy-haproxy + local resource=statefulset/upgrade-haproxy-haproxy + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-haproxy.yml + local new_result=/tmp/tmp.Zvl3zkw7nj/statefulset_upgrade-haproxy-haproxy.yml + desc 'compare statefulset/upgrade-haproxy-haproxy-' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/upgrade-haproxy-haproxy- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-haproxy-eks.yml ']' + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + version_gt 1.33 ++ echo '1.32 >= 1.33' ++ bc -l + '[' 0 -eq 1 ']' + return 1 + version_gt 1.29 ++ echo '1.32 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-haproxy-k129.yml ']' + version_gt 1.27 ++ echo '1.32 >= 1.27' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-haproxy-k127.yml ']' + version_gt 1.24 ++ echo '1.32 >= 1.24' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-haproxy-k124.yml ']' + version_gt 1.22 ++ echo '1.32 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-haproxy-k122.yml ']' + version_gt 1.21 ++ echo '1.32 >= 1.21' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-haproxy-k121.yml ']' + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-haproxy-oc.yml ']' + version_gt 1.29 ++ echo '1.32 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-haproxy-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-haproxy-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-haproxy-aks.yml ']' + kubectl_bin get -o yaml statefulset/upgrade-haproxy-haproxy ++ mktemp + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("upgrade-haproxy-28003", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + local LAST_OUT=/tmp/tmp.PmU1wmYULN ++ mktemp + local LAST_ERR=/tmp/tmp.xZKpA3Nc7A + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/upgrade-haproxy-haproxy + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PmU1wmYULN + cat /tmp/tmp.xZKpA3Nc7A + rm /tmp/tmp.PmU1wmYULN /tmp/tmp.xZKpA3Nc7A + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/statefulset_upgrade-haproxy-haproxy.yml /tmp/tmp.Zvl3zkw7nj/statefulset_upgrade-haproxy-haproxy.yml + log 'compare_kubectl: statefulset/upgrade-haproxy-haproxy OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-04-08T19:03:17+0000]' compare_kubectl: statefulset/upgrade-haproxy-haproxy OK [2026-04-08T19:03:17+0000] compare_kubectl: statefulset/upgrade-haproxy-haproxy OK + sleep 60 + desc 'check data exists after upgrade and drop database' + set +o xtrace ----------------------------------------------------------------------------------- check data exists after upgrade and drop database ----------------------------------------------------------------------------------- + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kvtnspcAeX +++ mktemp ++ local LAST_ERR=/tmp/tmp.1HeexuZxd5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kvtnspcAeX ++ cat /tmp/tmp.1HeexuZxd5 ++ rm /tmp/tmp.kvtnspcAeX /tmp/tmp.1HeexuZxd5 ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ echo pxc-client-6c975684f9-4j5ht ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Zvl3zkw7nj/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Zvl3zkw7nj/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.Zvl3zkw7nj/select-1.sql + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.P2YPVWPnXZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.XzkZ2i1Iwu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.P2YPVWPnXZ ++ cat /tmp/tmp.XzkZ2i1Iwu ++ rm /tmp/tmp.P2YPVWPnXZ /tmp/tmp.XzkZ2i1Iwu ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ echo pxc-client-6c975684f9-4j5ht ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Zvl3zkw7nj/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Zvl3zkw7nj/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.Zvl3zkw7nj/select-1.sql + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8eV6D6Eu22 +++ mktemp ++ local LAST_ERR=/tmp/tmp.DoSgc2wPFc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8eV6D6Eu22 ++ cat /tmp/tmp.DoSgc2wPFc ++ rm /tmp/tmp.8eV6D6Eu22 /tmp/tmp.DoSgc2wPFc ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ echo pxc-client-6c975684f9-4j5ht ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Zvl3zkw7nj/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Zvl3zkw7nj/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.Zvl3zkw7nj/select-1.sql + run_mysql 'drop database myApp;' '-h upgrade-haproxy-pxc -uroot -proot_password' + local 'command=drop database myApp;' + local 'uri=-h upgrade-haproxy-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Q7lq8ZI6N6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.70KrjAfUAr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Q7lq8ZI6N6 ++ cat /tmp/tmp.70KrjAfUAr ++ rm /tmp/tmp.Q7lq8ZI6N6 /tmp/tmp.70KrjAfUAr ++ return 0 + client_pod=pxc-client-6c975684f9-4j5ht + wait_pod pxc-client-6c975684f9-4j5ht + local pod=pxc-client-6c975684f9-4j5ht + local max_retry=480 + local ns= ++ echo pxc-client-6c975684f9-4j5ht ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6c975684f9-4j5ht condition met waiting for pod/pxc-client-6c975684f9-4j5ht to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + desc 'run restore' + set +o xtrace ----------------------------------------------------------------------------------- run restore ----------------------------------------------------------------------------------- + run_recovery_check upgrade-haproxy on-demand-backup-minio + local cluster=upgrade-haproxy + local backup=on-demand-backup-minio ++ get_proxy_engine upgrade-haproxy ++ local cluster_name=upgrade-haproxy +++ get_proxy upgrade-haproxy +++ local target_cluster=upgrade-haproxy ++++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.kd9ruAHaLa +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.Mkwoh6gTgD ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.kd9ruAHaLa ++++ cat /tmp/tmp.Mkwoh6gTgD ++++ rm /tmp/tmp.kd9ruAHaLa /tmp/tmp.Mkwoh6gTgD ++++ return 0 +++ [[ true == \t\r\u\e ]] +++ echo upgrade-haproxy-haproxy +++ return ++ local cluster_proxy=upgrade-haproxy-haproxy ++ echo haproxy + local proxy=haproxy ++ get_pvc_name_for_backup on-demand-backup-minio ++ local backup_name=on-demand-backup-minio +++ kubectl_bin get pxc-backup on-demand-backup-minio -o 'jsonpath={.status.destination}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.of57IEvWT5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.U7mn4HKGgu +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc-backup on-demand-backup-minio -o 'jsonpath={.status.destination}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.of57IEvWT5 +++ cat /tmp/tmp.U7mn4HKGgu +++ rm /tmp/tmp.of57IEvWT5 /tmp/tmp.U7mn4HKGgu +++ return 0 ++ local destination=s3://operator-testing/upgrade-haproxy-2026-04-08-18:56:18-full ++ [[ -z s3://operator-testing/upgrade-haproxy-2026-04-08-18:56:18-full ]] ++ local pvc_name=s3://operator-testing/upgrade-haproxy-2026-04-08-18:56:18-full ++ echo s3://operator-testing/upgrade-haproxy-2026-04-08-18:56:18-full + pvc_name=s3://operator-testing/upgrade-haproxy-2026-04-08-18:56:18-full + dest=pvc/s3://operator-testing/upgrade-haproxy-2026-04-08-18:56:18-full + log 'run pxc-restore/on-demand-backup-minio' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-04-08T19:04:32+0000]' run pxc-restore/on-demand-backup-minio [2026-04-08T19:04:32+0000] run pxc-restore/on-demand-backup-minio + YAML_INPUT_FILE=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/conf/restore-on-demand-backup-minio.yaml ++ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2428/e2e-tests/upgrade-haproxy/conf/restore-on-demand-backup-minio.yaml ++ /usr/bin/sed -e s~minio-service.#namespace~minio-service.upgrade-haproxy-28003~ ++ /usr/bin/sed -e 's##pvc/s3://operator-testing/upgrade-haproxy-2026-04-08-18:56:18-full#g' + MODIFIED_YAML='apiVersion: pxc.percona.com/v1 kind: PerconaXtraDBClusterRestore metadata: name: on-demand-backup-minio spec: pxcCluster: upgrade-haproxy backupName: on-demand-backup-minio' + [[ percona/percona-xtradb-cluster:5.7.44-31.65 =~ 5\.7 ]] + [[ on-demand-backup-minio == \o\n\-\d\e\m\a\n\d\-\b\a\c\k\u\p\-\a\w\s\-\s\3 ]] + FINAL_YAML='apiVersion: pxc.percona.com/v1 kind: PerconaXtraDBClusterRestore metadata: name: on-demand-backup-minio spec: pxcCluster: upgrade-haproxy backupName: on-demand-backup-minio' + kubectl_bin apply -f - + echo 'apiVersion: pxc.percona.com/v1 kind: PerconaXtraDBClusterRestore metadata: name: on-demand-backup-minio spec: pxcCluster: upgrade-haproxy backupName: on-demand-backup-minio' ++ mktemp + local LAST_OUT=/tmp/tmp.aNYaj6l3o1 ++ mktemp + local LAST_ERR=/tmp/tmp.sDpZxM5Iat + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.aNYaj6l3o1 perconaxtradbclusterrestore.pxc.percona.com/on-demand-backup-minio created + cat /tmp/tmp.sDpZxM5Iat + rm /tmp/tmp.aNYaj6l3o1 /tmp/tmp.sDpZxM5Iat + return 0 + wait_backup_restore on-demand-backup-minio + local backup_name=on-demand-backup-minio + local target_state=Succeeded + local wait_time=720 + set +o xtrace waiting for pxc-restore/on-demand-backup-minio to reach Succeeded state 2026-04-08T19:04:34 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:36 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:37 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:39 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:40 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:42 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:43 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:45 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:46 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:48 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:50 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:51 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:52 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:54 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:56 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:57 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:04:59 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:05:01 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:05:03 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:05:05 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:05:07 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:05:09 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:05:11 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:05:13 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:05:14 pxc-restore/on-demand-backup-minio state: Stopping Cluster 2026-04-08T19:05:15 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:17 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:19 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:20 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:22 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:23 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:24 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:26 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:27 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:29 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:30 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:31 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:33 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:35 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:37 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:39 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:41 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:43 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:44 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:46 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:48 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:50 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:52 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:54 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:56 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:05:58 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:06:00 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:06:02 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:06:04 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:06:06 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:06:08 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:06:10 pxc-restore/on-demand-backup-minio state: Restoring 2026-04-08T19:06:12 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:14 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:16 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:18 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:20 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:22 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:23 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:25 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:27 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:29 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:30 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:32 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:33 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:35 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:37 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:39 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:41 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:43 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:45 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:47 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:48 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:50 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:52 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:53 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:55 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:57 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:06:59 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:01 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:03 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:04 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:05 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:07 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:09 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:10 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:12 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:13 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:15 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:16 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:18 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:19 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:21 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:23 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:25 pxc-restore/on-demand-backup-minio state: Preparing Cluster 2026-04-08T19:07:27 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:29 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:30 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:31 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:33 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:35 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:37 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:39 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:40 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:41 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:43 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:45 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:46 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:48 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:49 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:51 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:53 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:55 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:57 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:58 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:07:59 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:01 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:03 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:05 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:07 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:09 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:11 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:12 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:13 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:15 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:17 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:18 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:20 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:22 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:23 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:24 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:26 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:28 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:29 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:31 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:32 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:34 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:35 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:37 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:39 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:41 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:42 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:44 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:46 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:47 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:49 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:51 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:52 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:53 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:55 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:57 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:08:59 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:00 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:02 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:04 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:05 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:08 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:10 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:11 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:13 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:15 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:17 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:19 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:21 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:23 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:25 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:26 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:28 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:29 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:31 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:33 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:35 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:37 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:39 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:41 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:42 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:44 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:46 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:47 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:49 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:51 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:52 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:54 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:56 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:58 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:09:59 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:01 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:03 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:05 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:07 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:09 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:10 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:12 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:14 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:15 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:17 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:19 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:21 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:23 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:25 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:26 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:27 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:29 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:31 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:33 pxc-restore/on-demand-backup-minio state: Starting Cluster 2026-04-08T19:10:35 pxc-restore/on-demand-backup-minio state: Starting Cluster