Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/logs/upgrade-haproxy-8-0.log + CLUSTER=upgrade-haproxy + CLUSTER_SIZE=3 + TARGET_OPERATOR_VER=1.13.0 + TARGET_IMAGE=perconalab/percona-xtradb-cluster-operator:PR-1131-d64e70d4 + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + TARGET_IMAGE_PMM=perconalab/pmm-client:dev-latest + TARGET_IMAGE_PROXY=perconalab/percona-xtradb-cluster-operator:main-proxysql + TARGET_IMAGE_HAPROXY=perconalab/percona-xtradb-cluster-operator:main-haproxy + TARGET_IMAGE_BACKUP=perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + PXC_VER=8.0 ++ tail -n1 ++ jq -r '.versions[].operator' ++ sort -V ++ curl -s https://check.percona.com/versions/v1/pxc-operator + INIT_OPERATOR_VER=1.12.0 + [[ 1.12.0 == \1\.\1\3\.\0 ]] + GIT_TAG=v1.12.0 ++ curl -s 'https://check.percona.com/versions/v1/pxc-operator/1.12.0/latest?databaseVersion=8.0' + INIT_OPERATOR_IMAGES='{"versions":[{"product":"pxc-operator", "operator":"1.12.0", "matrix":{"mongod":{}, "pxc":{"8.0.31-23.2":{"imagePath":"percona/percona-xtradb-cluster:8.0.31-23.2", "imageHash":"ed1ceea0b594ae34a92c891b4e42bc543d24999c82e47382cf53e33be4ae1d71", "status":"recommended", "critical":false}}, "pmm":{"2.32.0":{"imagePath":"percona/pmm-client:2.32.0", "imageHash":"ee2f3db541857e0a71633270596933441c4be579ce8e33c22cf150ead4f3622f", "status":"recommended", "critical":false}}, "proxysql":{"2.4.4-1.2":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-proxysql", "imageHash":"a0352f7491ba32f4cb1fb8edb1ad9248235f4ae1186932965383f47e3239e0f7", "status":"recommended", "critical":false}}, "haproxy":{"2.5.6":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-haproxy", "imageHash":"d900211bf5684839cfbaab3ec939ef7bae770638c7d819904820a3882a2aea32", "status":"recommended", "critical":false}}, "backup":{"8.0.32":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-pxc8.0.31-backup", "imageHash":"2eda0daac28521120fa262f7c6e8d3a24e92f0cf462837509a9b92b0dc05be8c", "status":"recommended", "critical":false}}, "operator":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0", "imageHash":"fbbf093bf7681e0784dd817ac311dea815140ce53aa9cdc79ada0bba067b06a8", "status":"recommended", "critical":false}}, "logCollector":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-logcollector", "imageHash":"6d4a4537fa766743c5aa6da8b683bf2656980d6979c31eab50a4be69c80f51ce", "status":"recommended", "critical":false}}, "postgresql":{}, "pgbackrest":{}, "pgbackrestRepo":{}, "pgbadger":{}, "pgbouncer":{}, "pxcOperator":{}, "psmdbOperator":{}, "pgOperatorApiserver":{}, "pgOperatorEvent":{}, "pgOperatorRmdata":{}, "pgOperatorScheduler":{}, "pgOperator":{}, "pgOperatorDeployer":{}, "psOperator":{}, "mysql":{}, "router":{}, "orchestrator":{}, "toolkit":{}}}]}' + OPERATOR_NAME=percona-xtradb-cluster-operator ++ echo '{"versions":[{"product":"pxc-operator", "operator":"1.12.0", "matrix":{"mongod":{}, "pxc":{"8.0.31-23.2":{"imagePath":"percona/percona-xtradb-cluster:8.0.31-23.2", "imageHash":"ed1ceea0b594ae34a92c891b4e42bc543d24999c82e47382cf53e33be4ae1d71", "status":"recommended", "critical":false}}, "pmm":{"2.32.0":{"imagePath":"percona/pmm-client:2.32.0", "imageHash":"ee2f3db541857e0a71633270596933441c4be579ce8e33c22cf150ead4f3622f", "status":"recommended", "critical":false}}, "proxysql":{"2.4.4-1.2":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-proxysql", "imageHash":"a0352f7491ba32f4cb1fb8edb1ad9248235f4ae1186932965383f47e3239e0f7", "status":"recommended", "critical":false}}, "haproxy":{"2.5.6":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-haproxy", "imageHash":"d900211bf5684839cfbaab3ec939ef7bae770638c7d819904820a3882a2aea32", "status":"recommended", "critical":false}}, "backup":{"8.0.32":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-pxc8.0.31-backup", "imageHash":"2eda0daac28521120fa262f7c6e8d3a24e92f0cf462837509a9b92b0dc05be8c", "status":"recommended", "critical":false}}, "operator":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0", "imageHash":"fbbf093bf7681e0784dd817ac311dea815140ce53aa9cdc79ada0bba067b06a8", "status":"recommended", "critical":false}}, "logCollector":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-logcollector", "imageHash":"6d4a4537fa766743c5aa6da8b683bf2656980d6979c31eab50a4be69c80f51ce", "status":"recommended", "critical":false}}, "postgresql":{}, "pgbackrest":{}, "pgbackrestRepo":{}, "pgbadger":{}, "pgbouncer":{}, "pxcOperator":{}, "psmdbOperator":{}, "pgOperatorApiserver":{}, "pgOperatorEvent":{}, "pgOperatorRmdata":{}, "pgOperatorScheduler":{}, "pgOperator":{}, "pgOperatorDeployer":{}, "psOperator":{}, "mysql":{}, "router":{}, "orchestrator":{}, "toolkit":{}}}]}' ++ jq -r '.versions[].matrix.operator[].imagePath' + IMAGE=percona/percona-xtradb-cluster-operator:1.12.0 ++ cut -d/ -f1 ++ echo perconalab/percona-xtradb-cluster-operator:PR-1131-d64e70d4 + [[ perconalab == \p\e\r\c\o\n\a\l\a\b ]] + IMAGE=perconalab/percona-xtradb-cluster-operator:1.12.0 ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ echo '{"versions":[{"product":"pxc-operator", "operator":"1.12.0", "matrix":{"mongod":{}, "pxc":{"8.0.31-23.2":{"imagePath":"percona/percona-xtradb-cluster:8.0.31-23.2", "imageHash":"ed1ceea0b594ae34a92c891b4e42bc543d24999c82e47382cf53e33be4ae1d71", "status":"recommended", "critical":false}}, "pmm":{"2.32.0":{"imagePath":"percona/pmm-client:2.32.0", "imageHash":"ee2f3db541857e0a71633270596933441c4be579ce8e33c22cf150ead4f3622f", "status":"recommended", "critical":false}}, "proxysql":{"2.4.4-1.2":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-proxysql", "imageHash":"a0352f7491ba32f4cb1fb8edb1ad9248235f4ae1186932965383f47e3239e0f7", "status":"recommended", "critical":false}}, "haproxy":{"2.5.6":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-haproxy", "imageHash":"d900211bf5684839cfbaab3ec939ef7bae770638c7d819904820a3882a2aea32", "status":"recommended", "critical":false}}, "backup":{"8.0.32":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-pxc8.0.31-backup", "imageHash":"2eda0daac28521120fa262f7c6e8d3a24e92f0cf462837509a9b92b0dc05be8c", "status":"recommended", "critical":false}}, "operator":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0", "imageHash":"fbbf093bf7681e0784dd817ac311dea815140ce53aa9cdc79ada0bba067b06a8", "status":"recommended", "critical":false}}, "logCollector":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-logcollector", "imageHash":"6d4a4537fa766743c5aa6da8b683bf2656980d6979c31eab50a4be69c80f51ce", "status":"recommended", "critical":false}}, "postgresql":{}, "pgbackrest":{}, "pgbackrestRepo":{}, "pgbadger":{}, "pgbouncer":{}, "pxcOperator":{}, "psmdbOperator":{}, "pgOperatorApiserver":{}, "pgOperatorEvent":{}, "pgOperatorRmdata":{}, "pgOperatorScheduler":{}, "pgOperator":{}, "pgOperatorDeployer":{}, "psOperator":{}, "mysql":{}, "router":{}, "orchestrator":{}, "toolkit":{}}}]}' + IMAGE_PXC=percona/percona-xtradb-cluster:8.0.31-23.2 ++ echo '{"versions":[{"product":"pxc-operator", "operator":"1.12.0", "matrix":{"mongod":{}, "pxc":{"8.0.31-23.2":{"imagePath":"percona/percona-xtradb-cluster:8.0.31-23.2", "imageHash":"ed1ceea0b594ae34a92c891b4e42bc543d24999c82e47382cf53e33be4ae1d71", "status":"recommended", "critical":false}}, "pmm":{"2.32.0":{"imagePath":"percona/pmm-client:2.32.0", "imageHash":"ee2f3db541857e0a71633270596933441c4be579ce8e33c22cf150ead4f3622f", "status":"recommended", "critical":false}}, "proxysql":{"2.4.4-1.2":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-proxysql", "imageHash":"a0352f7491ba32f4cb1fb8edb1ad9248235f4ae1186932965383f47e3239e0f7", "status":"recommended", "critical":false}}, "haproxy":{"2.5.6":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-haproxy", "imageHash":"d900211bf5684839cfbaab3ec939ef7bae770638c7d819904820a3882a2aea32", "status":"recommended", "critical":false}}, "backup":{"8.0.32":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-pxc8.0.31-backup", "imageHash":"2eda0daac28521120fa262f7c6e8d3a24e92f0cf462837509a9b92b0dc05be8c", "status":"recommended", "critical":false}}, "operator":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0", "imageHash":"fbbf093bf7681e0784dd817ac311dea815140ce53aa9cdc79ada0bba067b06a8", "status":"recommended", "critical":false}}, "logCollector":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-logcollector", "imageHash":"6d4a4537fa766743c5aa6da8b683bf2656980d6979c31eab50a4be69c80f51ce", "status":"recommended", "critical":false}}, "postgresql":{}, "pgbackrest":{}, "pgbackrestRepo":{}, "pgbadger":{}, "pgbouncer":{}, "pxcOperator":{}, "psmdbOperator":{}, "pgOperatorApiserver":{}, "pgOperatorEvent":{}, "pgOperatorRmdata":{}, "pgOperatorScheduler":{}, "pgOperator":{}, "pgOperatorDeployer":{}, "psOperator":{}, "mysql":{}, "router":{}, "orchestrator":{}, "toolkit":{}}}]}' ++ jq -r '.versions[].matrix.pmm[].imagePath' + IMAGE_PMM=percona/pmm-client:2.32.0 ++ jq -r '.versions[].matrix.proxysql[].imagePath' ++ echo '{"versions":[{"product":"pxc-operator", "operator":"1.12.0", "matrix":{"mongod":{}, "pxc":{"8.0.31-23.2":{"imagePath":"percona/percona-xtradb-cluster:8.0.31-23.2", "imageHash":"ed1ceea0b594ae34a92c891b4e42bc543d24999c82e47382cf53e33be4ae1d71", "status":"recommended", "critical":false}}, "pmm":{"2.32.0":{"imagePath":"percona/pmm-client:2.32.0", "imageHash":"ee2f3db541857e0a71633270596933441c4be579ce8e33c22cf150ead4f3622f", "status":"recommended", "critical":false}}, "proxysql":{"2.4.4-1.2":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-proxysql", "imageHash":"a0352f7491ba32f4cb1fb8edb1ad9248235f4ae1186932965383f47e3239e0f7", "status":"recommended", "critical":false}}, "haproxy":{"2.5.6":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-haproxy", "imageHash":"d900211bf5684839cfbaab3ec939ef7bae770638c7d819904820a3882a2aea32", "status":"recommended", "critical":false}}, "backup":{"8.0.32":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-pxc8.0.31-backup", "imageHash":"2eda0daac28521120fa262f7c6e8d3a24e92f0cf462837509a9b92b0dc05be8c", "status":"recommended", "critical":false}}, "operator":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0", "imageHash":"fbbf093bf7681e0784dd817ac311dea815140ce53aa9cdc79ada0bba067b06a8", "status":"recommended", "critical":false}}, "logCollector":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-logcollector", "imageHash":"6d4a4537fa766743c5aa6da8b683bf2656980d6979c31eab50a4be69c80f51ce", "status":"recommended", "critical":false}}, "postgresql":{}, "pgbackrest":{}, "pgbackrestRepo":{}, "pgbadger":{}, "pgbouncer":{}, "pxcOperator":{}, "psmdbOperator":{}, "pgOperatorApiserver":{}, "pgOperatorEvent":{}, "pgOperatorRmdata":{}, "pgOperatorScheduler":{}, "pgOperator":{}, "pgOperatorDeployer":{}, "psOperator":{}, "mysql":{}, "router":{}, "orchestrator":{}, "toolkit":{}}}]}' + IMAGE_PROXY=percona/percona-xtradb-cluster-operator:1.12.0-proxysql ++ jq -r '.versions[].matrix.haproxy[].imagePath' ++ echo '{"versions":[{"product":"pxc-operator", "operator":"1.12.0", "matrix":{"mongod":{}, "pxc":{"8.0.31-23.2":{"imagePath":"percona/percona-xtradb-cluster:8.0.31-23.2", "imageHash":"ed1ceea0b594ae34a92c891b4e42bc543d24999c82e47382cf53e33be4ae1d71", "status":"recommended", "critical":false}}, "pmm":{"2.32.0":{"imagePath":"percona/pmm-client:2.32.0", "imageHash":"ee2f3db541857e0a71633270596933441c4be579ce8e33c22cf150ead4f3622f", "status":"recommended", "critical":false}}, "proxysql":{"2.4.4-1.2":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-proxysql", "imageHash":"a0352f7491ba32f4cb1fb8edb1ad9248235f4ae1186932965383f47e3239e0f7", "status":"recommended", "critical":false}}, "haproxy":{"2.5.6":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-haproxy", "imageHash":"d900211bf5684839cfbaab3ec939ef7bae770638c7d819904820a3882a2aea32", "status":"recommended", "critical":false}}, "backup":{"8.0.32":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-pxc8.0.31-backup", "imageHash":"2eda0daac28521120fa262f7c6e8d3a24e92f0cf462837509a9b92b0dc05be8c", "status":"recommended", "critical":false}}, "operator":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0", "imageHash":"fbbf093bf7681e0784dd817ac311dea815140ce53aa9cdc79ada0bba067b06a8", "status":"recommended", "critical":false}}, "logCollector":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-logcollector", "imageHash":"6d4a4537fa766743c5aa6da8b683bf2656980d6979c31eab50a4be69c80f51ce", "status":"recommended", "critical":false}}, "postgresql":{}, "pgbackrest":{}, "pgbackrestRepo":{}, "pgbadger":{}, "pgbouncer":{}, "pxcOperator":{}, "psmdbOperator":{}, "pgOperatorApiserver":{}, "pgOperatorEvent":{}, "pgOperatorRmdata":{}, "pgOperatorScheduler":{}, "pgOperator":{}, "pgOperatorDeployer":{}, "psOperator":{}, "mysql":{}, "router":{}, "orchestrator":{}, "toolkit":{}}}]}' + IMAGE_HAPROXY=percona/percona-xtradb-cluster-operator:1.12.0-haproxy ++ echo '{"versions":[{"product":"pxc-operator", "operator":"1.12.0", "matrix":{"mongod":{}, "pxc":{"8.0.31-23.2":{"imagePath":"percona/percona-xtradb-cluster:8.0.31-23.2", "imageHash":"ed1ceea0b594ae34a92c891b4e42bc543d24999c82e47382cf53e33be4ae1d71", "status":"recommended", "critical":false}}, "pmm":{"2.32.0":{"imagePath":"percona/pmm-client:2.32.0", "imageHash":"ee2f3db541857e0a71633270596933441c4be579ce8e33c22cf150ead4f3622f", "status":"recommended", "critical":false}}, "proxysql":{"2.4.4-1.2":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-proxysql", "imageHash":"a0352f7491ba32f4cb1fb8edb1ad9248235f4ae1186932965383f47e3239e0f7", "status":"recommended", "critical":false}}, "haproxy":{"2.5.6":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-haproxy", "imageHash":"d900211bf5684839cfbaab3ec939ef7bae770638c7d819904820a3882a2aea32", "status":"recommended", "critical":false}}, "backup":{"8.0.32":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-pxc8.0.31-backup", "imageHash":"2eda0daac28521120fa262f7c6e8d3a24e92f0cf462837509a9b92b0dc05be8c", "status":"recommended", "critical":false}}, "operator":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0", "imageHash":"fbbf093bf7681e0784dd817ac311dea815140ce53aa9cdc79ada0bba067b06a8", "status":"recommended", "critical":false}}, "logCollector":{"1.12.0":{"imagePath":"percona/percona-xtradb-cluster-operator:1.12.0-logcollector", "imageHash":"6d4a4537fa766743c5aa6da8b683bf2656980d6979c31eab50a4be69c80f51ce", "status":"recommended", "critical":false}}, "postgresql":{}, "pgbackrest":{}, "pgbackrestRepo":{}, "pgbadger":{}, "pgbouncer":{}, "pxcOperator":{}, "psmdbOperator":{}, "pgOperatorApiserver":{}, "pgOperatorEvent":{}, "pgOperatorRmdata":{}, "pgOperatorScheduler":{}, "pgOperator":{}, "pgOperatorDeployer":{}, "psOperator":{}, "mysql":{}, "router":{}, "orchestrator":{}, "toolkit":{}}}]}' ++ jq -r '.versions[].matrix.backup[].imagePath' + IMAGE_BACKUP=percona/percona-xtradb-cluster-operator:1.12.0-pxc8.0.31-backup + [[ 1.13.0 == \1\.\1\2\.\0 ]] + main + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.6b1fuKzRAc ++ mktemp + local LAST_ERR=/tmp/tmp.OpkSMfidHQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6b1fuKzRAc namespace/cert-manager created + cat /tmp/tmp.OpkSMfidHQ + rm /tmp/tmp.6b1fuKzRAc /tmp/tmp.OpkSMfidHQ + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.OK3yU2LLMb ++ mktemp + local LAST_ERR=/tmp/tmp.MjfBxjiWYf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OK3yU2LLMb namespace/cert-manager labeled + cat /tmp/tmp.MjfBxjiWYf + rm /tmp/tmp.OK3yU2LLMb /tmp/tmp.MjfBxjiWYf + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.8.0/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.MShBGhupNM ++ mktemp + local LAST_ERR=/tmp/tmp.94OgHfJS0v + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.8.0/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MShBGhupNM namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created configmap/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews configured role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection configured rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.94OgHfJS0v Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.MShBGhupNM /tmp/tmp.94OgHfJS0v + return 0 + sleep 70 + create_infra_gh upgrade-haproxy-32496 v1.12.0 + local ns=upgrade-haproxy-32496 + local git_tag=v1.12.0 + '[' -n pxc-operator ']' + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + xargs kubectl delete ns + kubectl_bin get ns ++ mktemp + awk '{print$1}' + local LAST_OUT=/tmp/tmp.noLdhsQQes + local LAST_OUT=/tmp/tmp.5zysM9Wphh ++ mktemp + local LAST_ERR=/tmp/tmp.u9vIOjjJCa + local exit_status=0 ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.L4wcpIDVXD + local exit_status=0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5zysM9Wphh + cat /tmp/tmp.L4wcpIDVXD + rm /tmp/tmp.5zysM9Wphh /tmp/tmp.L4wcpIDVXD + return 0 namespace "cert-manager" deleted namespace "upgrade-haproxy-19268" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.noLdhsQQes namespace "pxc-operator" deleted + cat /tmp/tmp.u9vIOjjJCa + rm /tmp/tmp.noLdhsQQes /tmp/tmp.u9vIOjjJCa + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'namespace/pxc-operator - ' namespace/pxc-operator - + set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ma4oNG95In ++ mktemp + local LAST_ERR=/tmp/tmp.qtFrPPLnyk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ma4oNG95In namespace/pxc-operator created + cat /tmp/tmp.qtFrPPLnyk + rm /tmp/tmp.ma4oNG95In /tmp/tmp.qtFrPPLnyk + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.ALz2dmbwNv +++ mktemp ++ local LAST_ERR=/tmp/tmp.7AwExanpiU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ALz2dmbwNv ++ cat /tmp/tmp.7AwExanpiU ++ rm /tmp/tmp.ALz2dmbwNv /tmp/tmp.7AwExanpiU ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1131-d64e70d4-2-cluster5 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.21qAxK2bJf ++ mktemp + local LAST_ERR=/tmp/tmp.VIBnZFhYvg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1131-d64e70d4-2-cluster5 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.21qAxK2bJf Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1131-d64e70d4-2-cluster5" modified. + cat /tmp/tmp.VIBnZFhYvg + rm /tmp/tmp.21qAxK2bJf /tmp/tmp.VIBnZFhYvg + return 0 + deploy_operator_gh v1.12.0 + local git_tag=v1.12.0 + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- ++ kubectl_bin get crds -o 'jsonpath={.items[?(@.metadata.name == "perconaxtradbclusters.pxc.percona.com")].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4oVQMpBJz8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.zRaqvMcXMH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crds -o 'jsonpath={.items[?(@.metadata.name == "perconaxtradbclusters.pxc.percona.com")].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4oVQMpBJz8 ++ cat /tmp/tmp.zRaqvMcXMH ++ rm /tmp/tmp.4oVQMpBJz8 /tmp/tmp.zRaqvMcXMH ++ return 0 + [[ -n perconaxtradbclusters.pxc.percona.com ]] ++ kubectl_bin get crd/perconaxtradbclusters.pxc.percona.com -o 'jsonpath={.spec.versions[?(@.name == "v1-12-0")].name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AsoAbQq8dL +++ mktemp ++ local LAST_ERR=/tmp/tmp.N9tq7xZsfP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/perconaxtradbclusters.pxc.percona.com -o 'jsonpath={.spec.versions[?(@.name == "v1-12-0")].name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AsoAbQq8dL ++ cat /tmp/tmp.N9tq7xZsfP ++ rm /tmp/tmp.AsoAbQq8dL /tmp/tmp.N9tq7xZsfP ++ return 0 + [[ -n '' ]] + kubectl_bin apply --server-side --force-conflicts -f https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.12.0/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.EXZ09zgHXY ++ mktemp + local LAST_ERR=/tmp/tmp.cr7JKF1KGi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.12.0/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EXZ09zgHXY + cat /tmp/tmp.cr7JKF1KGi + rm /tmp/tmp.EXZ09zgHXY /tmp/tmp.cr7JKF1KGi + return 0 + local rbac_yaml=rbac + local operator_yaml=operator.yaml + '[' -n pxc-operator ']' + rbac_yaml=cw-rbac + operator_yaml=cw-operator.yaml + apply_rbac_gh cw-rbac v1.12.0 + local operator_namespace=pxc-operator + local rbac=cw-rbac + local git_tag=v1.12.0 + curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.12.0/deploy/cw-rbac.yaml + /usr/bin/sed -i -e 's^namespace: .*^namespace: pxc-operator^' /tmp/tmp.hd9zbR3qhS/rbac_v1.12.0.yaml + kubectl_bin apply -f /tmp/tmp.hd9zbR3qhS/rbac_v1.12.0.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.9yxYraIGEv ++ mktemp + local LAST_ERR=/tmp/tmp.Jb0oOdkl6C + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /tmp/tmp.hd9zbR3qhS/rbac_v1.12.0.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9yxYraIGEv clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.Jb0oOdkl6C + rm /tmp/tmp.9yxYraIGEv /tmp/tmp.Jb0oOdkl6C + return 0 + curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.12.0/deploy/cw-operator.yaml + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' + kubectl_bin apply -n pxc-operator -f - + cat /tmp/tmp.hd9zbR3qhS/cw-operator.yaml_v1.12.0.yaml ++ mktemp + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:1.12.0^' + local LAST_OUT=/tmp/tmp.SvdPlFkeEc ++ mktemp + local LAST_ERR=/tmp/tmp.FqqtMG311Q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n pxc-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SvdPlFkeEc deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.FqqtMG311Q + rm /tmp/tmp.SvdPlFkeEc /tmp/tmp.FqqtMG311Q + return 0 + sleep 2 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.2v9D3PlfIs +++ mktemp ++ local LAST_ERR=/tmp/tmp.winKnPqwUg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2v9D3PlfIs ++ cat /tmp/tmp.winKnPqwUg ++ rm /tmp/tmp.2v9D3PlfIs /tmp/tmp.winKnPqwUg ++ return 0 + wait_pod percona-xtradb-cluster-operator-57b76768bc-nkpmm + local pod=percona-xtradb-cluster-operator-57b76768bc-nkpmm + local max_retry=480 + local ns= ++ echo percona-xtradb-cluster-operator-57b76768bc-nkpmm ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace percona-xtradb-cluster-operator-57b76768bc-nkpmm.Ok + create_namespace upgrade-haproxy-32496 + local namespace=upgrade-haproxy-32496 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces upgrade-haproxy-32496' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces upgrade-haproxy-32496 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace upgrade-haproxy-32496 ++ mktemp + awk '{print$1}' + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.vQRQ3hrNpc + local LAST_OUT=/tmp/tmp.QaybZnvzuH ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.gz8gv7i4PB + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.pICwDCbfWi + local exit_status=0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace upgrade-haproxy-32496 ++ seq 0 2 + xargs kubectl delete ns + for i in '$(seq 0 2)' + set +e + kubectl get ns + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QaybZnvzuH + cat /tmp/tmp.pICwDCbfWi + rm /tmp/tmp.QaybZnvzuH /tmp/tmp.pICwDCbfWi + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace upgrade-haproxy-32496 Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace upgrade-haproxy-32496 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.vQRQ3hrNpc + cat /tmp/tmp.gz8gv7i4PB Error from server (NotFound): namespaces "upgrade-haproxy-32496" not found + rm /tmp/tmp.vQRQ3hrNpc /tmp/tmp.gz8gv7i4PB + return 1 + : + wait_for_delete namespace/upgrade-haproxy-32496 + local res=namespace/upgrade-haproxy-32496 + echo -n 'namespace/upgrade-haproxy-32496 - ' namespace/upgrade-haproxy-32496 - + set +o xtrace Error from server (NotFound): namespaces "upgrade-haproxy-32496" not found + desc 'create namespace upgrade-haproxy-32496' + set +o xtrace ----------------------------------------------------------------------------------- create namespace upgrade-haproxy-32496 ----------------------------------------------------------------------------------- + kubectl_bin create namespace upgrade-haproxy-32496 ++ mktemp + local LAST_OUT=/tmp/tmp.HH30KKYvWq ++ mktemp + local LAST_ERR=/tmp/tmp.bib5UpgwQc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace upgrade-haproxy-32496 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HH30KKYvWq namespace/upgrade-haproxy-32496 created + cat /tmp/tmp.bib5UpgwQc + rm /tmp/tmp.HH30KKYvWq /tmp/tmp.bib5UpgwQc + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.99Ee9S1qaz +++ mktemp ++ local LAST_ERR=/tmp/tmp.YDSqjZL56p ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.99Ee9S1qaz ++ cat /tmp/tmp.YDSqjZL56p ++ rm /tmp/tmp.99Ee9S1qaz /tmp/tmp.YDSqjZL56p ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1131-d64e70d4-2-cluster5 --namespace=upgrade-haproxy-32496 ++ mktemp + local LAST_OUT=/tmp/tmp.dpxMJQ3I9T ++ mktemp + local LAST_ERR=/tmp/tmp.i7BggzvA5j + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1131-d64e70d4-2-cluster5 --namespace=upgrade-haproxy-32496 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dpxMJQ3I9T Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1131-d64e70d4-2-cluster5" modified. + cat /tmp/tmp.i7BggzvA5j + rm /tmp/tmp.dpxMJQ3I9T /tmp/tmp.i7BggzvA5j + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.SW7Pjtr2eD ++ mktemp + local LAST_ERR=/tmp/tmp.aiXM5xNVoj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SW7Pjtr2eD secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.aiXM5xNVoj + rm /tmp/tmp.SW7Pjtr2eD /tmp/tmp.aiXM5xNVoj + return 0 + start_minio + deploy_helm upgrade-haproxy-32496 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add percona https://percona-charts.storage.googleapis.com/ "percona" already exists with the same configuration, skipping + helm repo add minio https://helm.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "stable" chart repository Update Complete. ⎈Happy Helming!⎈ + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 8.0.5 --set accessKey=some-access-key --set secretKey=some-secret-key --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set securityContext.enabled=false --set persistence.size=2G --set environment.MINIO_REGION=us-east-1 --set environment.MINIO_HTTP_TRACE=/tmp/trace.log minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 8.0.5 --set accessKey=some-access-key --set secretKey=some-secret-key --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set securityContext.enabled=false --set persistence.size=2G --set environment.MINIO_REGION=us-east-1 --set environment.MINIO_HTTP_TRACE=/tmp/trace.log minio/minio NAME: minio-service LAST DEPLOYED: Thu Apr 20 12:11:11 2023 NAMESPACE: upgrade-haproxy-32496 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: Minio can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.upgrade-haproxy-32496.svc.cluster.local To access Minio from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace upgrade-haproxy-32496 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace upgrade-haproxy-32496 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access Minio server on http://localhost:9000. Follow the below steps to connect to Minio server with mc client: 1. Download the Minio mc client - https://docs.minio.io/docs/minio-client-quickstart-guide 2. Get the ACCESS_KEY=$(kubectl get secret minio-service -o jsonpath="{.data.accesskey}" | base64 --decode) and the SECRET_KEY=$(kubectl get secret minio-service -o jsonpath="{.data.secretkey}" | base64 --decode) 3. mc alias set minio-service-local http://localhost:9000 "$ACCESS_KEY" "$SECRET_KEY" --api s3v4 4. mc ls minio-service-local Alternately, you can use your browser or the Minio SDK to access the server - https://docs.minio.io/categories/17 + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GdjvxVUvhb +++ mktemp ++ local LAST_ERR=/tmp/tmp.xBWIxz9rv8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GdjvxVUvhb ++ cat /tmp/tmp.xBWIxz9rv8 ++ rm /tmp/tmp.GdjvxVUvhb /tmp/tmp.xBWIxz9rv8 ++ return 0 + MINIO_POD=minio-service-569fc47cb6-zcjkw + wait_pod minio-service-569fc47cb6-zcjkw + local pod=minio-service-569fc47cb6-zcjkw + local max_retry=480 + local ns= ++ echo minio-service-569fc47cb6-zcjkw ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace minio-service-569fc47cb6-zcjkw.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.PrrFGIKnf4 ++ mktemp + local LAST_ERR=/tmp/tmp.jKjliGgONX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PrrFGIKnf4 make_bucket: operator-testing pod "aws-cli" deleted + cat /tmp/tmp.jKjliGgONX If you don't see a command prompt, try pressing enter. Error attaching, falling back to logs: unable to upgrade connection: container aws-cli not found in pod aws-cli_upgrade-haproxy-32496 + rm /tmp/tmp.PrrFGIKnf4 /tmp/tmp.jKjliGgONX + return 0 + local proxy=haproxy + local cr_yaml=/tmp/tmp.hd9zbR3qhS/cr_1.12.0_haproxy.yaml + prepare_cr_yaml /tmp/tmp.hd9zbR3qhS/cr_1.12.0_haproxy.yaml haproxy upgrade-haproxy 3 v1.12.0 + local cr_yaml=/tmp/tmp.hd9zbR3qhS/cr_1.12.0_haproxy.yaml + local proxy=haproxy + local cluster=upgrade-haproxy + local cluster_size=3 + local git_tag=v1.12.0 + curl -s https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/v1.12.0/deploy/cr.yaml + yq eval ' .metadata.name = "upgrade-haproxy" | .spec.secretsName = "my-cluster-secrets" | .spec.vaultSecretName = "some-name-vault" | .spec.sslSecretName = "some-name-ssl" | .spec.sslInternalSecretName = "some-name-ssl-internal" | .spec.upgradeOptions.apply = "disabled" | .spec.pxc.size = 3 | .spec.proxysql.size = 3 | .spec.haproxy.size = 3 | .spec.pxc.image = "-pxc" | .spec.proxysql.image = "-proxysql" | .spec.haproxy.image = "-haproxy" | .spec.backup.image = "-backup" | .spec.backup.storages.minio.s3.credentialsSecret = "minio-secret" | .spec.backup.storages.minio.s3.region = "us-east-1" | .spec.backup.storages.minio.s3.bucket = "operator-testing" | .spec.backup.storages.minio.s3.endpointUrl = "http://minio-service:9000/" | .spec.backup.storages.minio.type = "s3" ' - + [[ haproxy == \h\a\p\r\o\x\y ]] + yq -i eval ' .spec.haproxy.enabled = true | .spec.proxysql.enabled = false ' /tmp/tmp.hd9zbR3qhS/cr_1.12.0_haproxy.yaml + spinup_pxc upgrade-haproxy /tmp/tmp.hd9zbR3qhS/cr_1.12.0_haproxy.yaml 3 30 /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/conf/secrets_without_tls.yml + local cluster=upgrade-haproxy + local config=/tmp/tmp.hd9zbR3qhS/cr_1.12.0_haproxy.yaml + local size=3 + local sleep=30 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/conf/secrets_without_tls.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/conf/secrets_without_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.B82eyQv0u2 ++ mktemp + local LAST_ERR=/tmp/tmp.ffTRCFXnnp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/conf/secrets_without_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.B82eyQv0u2 secret/my-cluster-secrets created + cat /tmp/tmp.ffTRCFXnnp + rm /tmp/tmp.B82eyQv0u2 /tmp/tmp.ffTRCFXnnp + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/conf/client.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_OUT=/tmp/tmp.PoFfmdEbUJ + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.31-23.2#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:1.12.0#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.upgrade-haproxy-32496~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: percona/percona-xtradb-cluster-operator:1.12.0-haproxy#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.31-23.2#' + /usr/bin/sed -e 's#image:.*-pmm$#image: percona/pmm-client:2.32.0#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: percona/percona-xtradb-cluster-operator:1.12.0-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: percona/percona-xtradb-cluster-operator:1.12.0-pxc8.0.31-backup#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/conf/client.yml ++ mktemp + local LAST_ERR=/tmp/tmp.Xqxw3uu13z + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PoFfmdEbUJ deployment.apps/pxc-client created + cat /tmp/tmp.Xqxw3uu13z + rm /tmp/tmp.PoFfmdEbUJ /tmp/tmp.Xqxw3uu13z + return 0 + [[ percona/percona-xtradb-cluster:8.0.31-23.2 =~ 5\.7 ]] + apply_config /tmp/tmp.hd9zbR3qhS/cr_1.12.0_haproxy.yaml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.hd9zbR3qhS/cr_1.12.0_haproxy.yaml + /usr/bin/sed -e 's#image:.*-pmm$#image: percona/pmm-client:2.32.0#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:1.12.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.31-23.2#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.upgrade-haproxy-32496~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + cat /tmp/tmp.hd9zbR3qhS/cr_1.12.0_haproxy.yaml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.31-23.2#' + /usr/bin/sed -e 's#image:.*-backup$#image: percona/percona-xtradb-cluster-operator:1.12.0-pxc8.0.31-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: percona/percona-xtradb-cluster-operator:1.12.0-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: percona/percona-xtradb-cluster-operator:1.12.0-haproxy#' ++ mktemp + local LAST_OUT=/tmp/tmp.w85InwV0wj ++ mktemp + local LAST_ERR=/tmp/tmp.Bp23R1EDPU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.w85InwV0wj perconaxtradbcluster.pxc.percona.com/upgrade-haproxy created + cat /tmp/tmp.Bp23R1EDPU + rm /tmp/tmp.w85InwV0wj /tmp/tmp.Bp23R1EDPU + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy upgrade-haproxy ++ local target_cluster=upgrade-haproxy +++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tyuXjO68jT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rimvMlkGQA +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.tyuXjO68jT +++ cat /tmp/tmp.rimvMlkGQA +++ rm /tmp/tmp.tyuXjO68jT /tmp/tmp.rimvMlkGQA +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo upgrade-haproxy-haproxy ++ return + local proxy=upgrade-haproxy-haproxy + wait_for_running upgrade-haproxy-haproxy 1 + local name=upgrade-haproxy-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-haproxy-0 480 + local pod=upgrade-haproxy-haproxy-0 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-haproxy-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace upgrade-haproxy-haproxy-0....................error: a container name must be specified for pod upgrade-haproxy-haproxy-0, choose one of: [haproxy pxc-monit] .Ok + wait_for_running upgrade-haproxy-pxc 3 + local name=upgrade-haproxy-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-0 480 + local pod=upgrade-haproxy-pxc-0 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo upgrade-haproxy-pxc-0 + local container=pxc + set +o xtrace upgrade-haproxy-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-1 480 + local pod=upgrade-haproxy-pxc-1 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-pxc-1 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace upgrade-haproxy-pxc-1................Ok + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-2 480 + local pod=upgrade-haproxy-pxc-2 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace upgrade-haproxy-pxc-2...................Ok + sleep 30 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.31-23.2 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h upgrade-haproxy-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h upgrade-haproxy-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DGmDKQnKtX +++ mktemp ++ local LAST_ERR=/tmp/tmp.AbADCEn8Ar ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DGmDKQnKtX ++ cat /tmp/tmp.AbADCEn8Ar ++ rm /tmp/tmp.DGmDKQnKtX /tmp/tmp.AbADCEn8Ar ++ return 0 + client_pod=pxc-client-7c98b85c48-9n2xs + wait_pod pxc-client-7c98b85c48-9n2xs + local pod=pxc-client-7c98b85c48-9n2xs + local max_retry=480 + local ns= ++ echo pxc-client-7c98b85c48-9n2xs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-7c98b85c48-9n2xs.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h upgrade-haproxy-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h upgrade-haproxy-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TdDQWlnEb8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.7BS0lQ1nWr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TdDQWlnEb8 ++ cat /tmp/tmp.7BS0lQ1nWr ++ rm /tmp/tmp.TdDQWlnEb8 /tmp/tmp.7BS0lQ1nWr ++ return 0 + client_pod=pxc-client-7c98b85c48-9n2xs + wait_pod pxc-client-7c98b85c48-9n2xs + local pod=pxc-client-7c98b85c48-9n2xs + local max_retry=480 + local ns= ++ echo pxc-client-7c98b85c48-9n2xs ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-7c98b85c48-9n2xs.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.31-23.2 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.L2HqnQeEui +++ mktemp ++ local LAST_ERR=/tmp/tmp.6qEViebVZN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.L2HqnQeEui ++ cat /tmp/tmp.6qEViebVZN ++ rm /tmp/tmp.L2HqnQeEui /tmp/tmp.6qEViebVZN ++ return 0 + client_pod=pxc-client-7c98b85c48-9n2xs + wait_pod pxc-client-7c98b85c48-9n2xs + local pod=pxc-client-7c98b85c48-9n2xs + local max_retry=480 + local ns= ++ echo pxc-client-7c98b85c48-9n2xs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-7c98b85c48-9n2xs.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.hd9zbR3qhS/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.hd9zbR3qhS/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.31-23.2 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wUCgcGRqHg +++ mktemp ++ local LAST_ERR=/tmp/tmp.sSydUQ7Xca ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wUCgcGRqHg ++ cat /tmp/tmp.sSydUQ7Xca ++ rm /tmp/tmp.wUCgcGRqHg /tmp/tmp.sSydUQ7Xca ++ return 0 + client_pod=pxc-client-7c98b85c48-9n2xs + wait_pod pxc-client-7c98b85c48-9n2xs + local pod=pxc-client-7c98b85c48-9n2xs + local max_retry=480 + local ns= ++ echo pxc-client-7c98b85c48-9n2xs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-7c98b85c48-9n2xs.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.hd9zbR3qhS/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.hd9zbR3qhS/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.31-23.2 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HGhWwyEzlI +++ mktemp ++ local LAST_ERR=/tmp/tmp.nndtn3STzg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HGhWwyEzlI ++ cat /tmp/tmp.nndtn3STzg ++ rm /tmp/tmp.HGhWwyEzlI /tmp/tmp.nndtn3STzg ++ return 0 + client_pod=pxc-client-7c98b85c48-9n2xs + wait_pod pxc-client-7c98b85c48-9n2xs + local pod=pxc-client-7c98b85c48-9n2xs + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-7c98b85c48-9n2xs ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-7c98b85c48-9n2xs.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.hd9zbR3qhS/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.hd9zbR3qhS/select-1.sql ++ is_keyring_plugin_in_use upgrade-haproxy ++ local cluster=upgrade-haproxy ++ kubectl_bin exec -it upgrade-haproxy-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kUukNw14KR +++ mktemp ++ egrep -o 'early-plugin-load=keyring_\w+.so' ++ local LAST_ERR=/tmp/tmp.VZdS3WJyDD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it upgrade-haproxy-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kUukNw14KR ++ cat /tmp/tmp.VZdS3WJyDD Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.kUukNw14KR /tmp/tmp.VZdS3WJyDD ++ return 0 + '[' '' ']' + compare_generation 1 haproxy upgrade-haproxy + local generation=1 + local proxy=haproxy + local cluster=upgrade-haproxy + local current_generation + [[ haproxy == \h\a\p\r\o\x\y ]] + containers=(pxc haproxy) + for container in '"${containers[@]}"' ++ kubectl_bin get statefulset upgrade-haproxy-pxc -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w5hE3EhCRn +++ mktemp ++ local LAST_ERR=/tmp/tmp.bIjOTHlSRy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get statefulset upgrade-haproxy-pxc -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.w5hE3EhCRn ++ cat /tmp/tmp.bIjOTHlSRy ++ rm /tmp/tmp.w5hE3EhCRn /tmp/tmp.bIjOTHlSRy ++ return 0 + current_generation=1 + [[ 1 != \1 ]] + for container in '"${containers[@]}"' ++ kubectl_bin get statefulset upgrade-haproxy-haproxy -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EuxwmzGQdE +++ mktemp ++ local LAST_ERR=/tmp/tmp.QZ55CrFqSy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get statefulset upgrade-haproxy-haproxy -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EuxwmzGQdE ++ cat /tmp/tmp.QZ55CrFqSy ++ rm /tmp/tmp.EuxwmzGQdE /tmp/tmp.QZ55CrFqSy ++ return 0 + current_generation=1 + [[ 1 != \1 ]] + run_backup upgrade-haproxy on-demand-backup-minio + local cluster=upgrade-haproxy + local backup1=on-demand-backup-minio + desc 'make backup on-demand-backup-minio' + set +o xtrace ----------------------------------------------------------------------------------- make backup on-demand-backup-minio ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/conf/on-demand-backup-minio.yml ++ mktemp + local LAST_OUT=/tmp/tmp.nHc3WIFWDd ++ mktemp + local LAST_ERR=/tmp/tmp.6YTlS48tVZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/conf/on-demand-backup-minio.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nHc3WIFWDd perconaxtradbclusterbackup.pxc.percona.com/on-demand-backup-minio created + cat /tmp/tmp.6YTlS48tVZ + rm /tmp/tmp.nHc3WIFWDd /tmp/tmp.6YTlS48tVZ + return 0 + wait_backup on-demand-backup-minio + local backup=on-demand-backup-minio + local status=Succeeded + set +o xtrace on-demand-backup-minio........Succeeded + desc 'upgrade operator' + set +o xtrace ----------------------------------------------------------------------------------- upgrade operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.fu009exCgQ ++ mktemp + local LAST_ERR=/tmp/tmp.xBqx7N2U3C + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fu009exCgQ customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.xBqx7N2U3C + rm /tmp/tmp.fu009exCgQ /tmp/tmp.xBqx7N2U3C + return 0 + [[ -n pxc-operator ]] + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/deploy/cw-rbac.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.z9OPoShbNQ ++ mktemp + local LAST_ERR=/tmp/tmp.pZXlQ2HlMW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.z9OPoShbNQ clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.pZXlQ2HlMW + rm /tmp/tmp.z9OPoShbNQ /tmp/tmp.pZXlQ2HlMW + return 0 + kubectl_bin patch deployment percona-xtradb-cluster-operator '-p{"spec":{"template":{"spec":{"containers":[{"name":"percona-xtradb-cluster-operator","image":"perconalab/percona-xtradb-cluster-operator:PR-1131-d64e70d4"}]}}}}' -n pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.Oel4azSwu1 ++ mktemp + local LAST_ERR=/tmp/tmp.xsqaAHLXZy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch deployment percona-xtradb-cluster-operator '-p{"spec":{"template":{"spec":{"containers":[{"name":"percona-xtradb-cluster-operator","image":"perconalab/percona-xtradb-cluster-operator:PR-1131-d64e70d4"}]}}}}' -n pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Oel4azSwu1 deployment.apps/percona-xtradb-cluster-operator patched + cat /tmp/tmp.xsqaAHLXZy + rm /tmp/tmp.Oel4azSwu1 /tmp/tmp.xsqaAHLXZy + return 0 + kubectl_bin rollout status deployment/percona-xtradb-cluster-operator -n pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.fUFhzSDpfs ++ mktemp + local LAST_ERR=/tmp/tmp.RF6u13EZeS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl rollout status deployment/percona-xtradb-cluster-operator -n pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fUFhzSDpfs Waiting for deployment "percona-xtradb-cluster-operator" rollout to finish: 0 of 1 updated replicas are available... deployment "percona-xtradb-cluster-operator" successfully rolled out + cat /tmp/tmp.RF6u13EZeS + rm /tmp/tmp.fUFhzSDpfs /tmp/tmp.RF6u13EZeS + return 0 + sleep 10 + desc 'wait for operator upgrade' + set +o xtrace ----------------------------------------------------------------------------------- wait for operator upgrade ----------------------------------------------------------------------------------- ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image' -n pxc-operator ++ grep -vc NAME ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WkY2FiXOl7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.LjxGXCcXAX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WkY2FiXOl7 ++ cat /tmp/tmp.LjxGXCcXAX ++ rm /tmp/tmp.WkY2FiXOl7 /tmp/tmp.LjxGXCcXAX ++ return 0 + [[ 1 -eq 1 ]] + '[' -n pxc-operator ']' ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.E2XLKX0WeX +++ mktemp ++ local LAST_ERR=/tmp/tmp.hxq5xJ4RDH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.E2XLKX0WeX ++ cat /tmp/tmp.hxq5xJ4RDH ++ rm /tmp/tmp.E2XLKX0WeX /tmp/tmp.hxq5xJ4RDH ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1131-d64e70d4-2-cluster5 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.MpV8pQ0DEO ++ mktemp + local LAST_ERR=/tmp/tmp.91tsJsmhdB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1131-d64e70d4-2-cluster5 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MpV8pQ0DEO Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1131-d64e70d4-2-cluster5" modified. + cat /tmp/tmp.91tsJsmhdB + rm /tmp/tmp.MpV8pQ0DEO /tmp/tmp.91tsJsmhdB + return 0 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image' ++ grep perconalab/percona-xtradb-cluster-operator:PR-1131-d64e70d4 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yOkpv6l8UW +++ mktemp ++ local LAST_ERR=/tmp/tmp.KXyFGxcKNC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yOkpv6l8UW ++ cat /tmp/tmp.KXyFGxcKNC ++ rm /tmp/tmp.yOkpv6l8UW /tmp/tmp.KXyFGxcKNC ++ return 0 + wait_pod percona-xtradb-cluster-operator-79d989c88f-wn4wv + local pod=percona-xtradb-cluster-operator-79d989c88f-wn4wv + local max_retry=480 + local ns= ++ echo percona-xtradb-cluster-operator-79d989c88f-wn4wv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace percona-xtradb-cluster-operator-79d989c88f-wn4wv...Ok ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.arlgJbVmZk +++ mktemp ++ local LAST_ERR=/tmp/tmp.mKJa09r1ES ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.arlgJbVmZk ++ cat /tmp/tmp.mKJa09r1ES ++ rm /tmp/tmp.arlgJbVmZk /tmp/tmp.mKJa09r1ES ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1131-d64e70d4-2-cluster5 --namespace=upgrade-haproxy-32496 ++ mktemp + local LAST_OUT=/tmp/tmp.Qh2cppKfmS ++ mktemp + local LAST_ERR=/tmp/tmp.HwaSKHTITc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1131-d64e70d4-2-cluster5 --namespace=upgrade-haproxy-32496 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Qh2cppKfmS Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1131-d64e70d4-2-cluster5" modified. + cat /tmp/tmp.HwaSKHTITc + rm /tmp/tmp.Qh2cppKfmS /tmp/tmp.HwaSKHTITc + return 0 + desc 'check images and generation after operator upgrade' + set +o xtrace ----------------------------------------------------------------------------------- check images and generation after operator upgrade ----------------------------------------------------------------------------------- + check_pxc_liveness upgrade-haproxy 3 + local cluster=upgrade-haproxy + local cluster_size=3 + wait_cluster_consistency upgrade-haproxy 3 + local cluster_name=upgrade-haproxy + local cluster_size=3 + local proxy_size= + '[' -z '' ']' + proxy_size=3 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qkUHCVQYC3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.EcFG60hVLw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qkUHCVQYC3 ++ cat /tmp/tmp.EcFG60hVLw ++ rm /tmp/tmp.qkUHCVQYC3 /tmp/tmp.EcFG60hVLw ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Qm3Knfer2J +++ mktemp ++ local LAST_ERR=/tmp/tmp.W3JbEPJeBd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Qm3Knfer2J ++ cat /tmp/tmp.W3JbEPJeBd ++ rm /tmp/tmp.Qm3Knfer2J /tmp/tmp.W3JbEPJeBd ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine upgrade-haproxy +++ local cluster_name=upgrade-haproxy ++++ get_proxy upgrade-haproxy ++++ local target_cluster=upgrade-haproxy +++++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.tZZMmLqRcC ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.oEKnnFOYjL +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.tZZMmLqRcC +++++ cat /tmp/tmp.oEKnnFOYjL +++++ rm /tmp/tmp.tZZMmLqRcC /tmp/tmp.oEKnnFOYjL +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo upgrade-haproxy-haproxy ++++ return +++ local cluster_proxy=upgrade-haproxy-haproxy +++ echo haproxy ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kJsTlVGkaC +++ mktemp ++ local LAST_ERR=/tmp/tmp.rWR1NKaAp1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kJsTlVGkaC ++ cat /tmp/tmp.rWR1NKaAp1 ++ rm /tmp/tmp.kJsTlVGkaC /tmp/tmp.rWR1NKaAp1 ++ return 0 + [[ 3 == \3 ]] + wait_for_running upgrade-haproxy-pxc 3 + local name=upgrade-haproxy-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-0 480 + local pod=upgrade-haproxy-pxc-0 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace upgrade-haproxy-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-1 480 + local pod=upgrade-haproxy-pxc-1 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace upgrade-haproxy-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod upgrade-haproxy-pxc-2 480 + local pod=upgrade-haproxy-pxc-2 + local max_retry=480 + local ns= ++ echo upgrade-haproxy-pxc-2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace upgrade-haproxy-pxc-2.Ok ++ seq 0 2 + for i in '$(seq 0 $((cluster_size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.31-23.2 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-0.upgrade-haproxy-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vyGDzrgMj0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.LHExMFFn01 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vyGDzrgMj0 ++ cat /tmp/tmp.LHExMFFn01 ++ rm /tmp/tmp.vyGDzrgMj0 /tmp/tmp.LHExMFFn01 ++ return 0 + client_pod=pxc-client-7c98b85c48-9n2xs + wait_pod pxc-client-7c98b85c48-9n2xs + local pod=pxc-client-7c98b85c48-9n2xs + local max_retry=480 + local ns= ++ echo pxc-client-7c98b85c48-9n2xs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-7c98b85c48-9n2xs.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.hd9zbR3qhS/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.hd9zbR3qhS/select-1.sql + for i in '$(seq 0 $((cluster_size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.31-23.2 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-1.upgrade-haproxy-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ssx8WvfRCs +++ mktemp ++ local LAST_ERR=/tmp/tmp.ILbukw6pEp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ssx8WvfRCs ++ cat /tmp/tmp.ILbukw6pEp ++ rm /tmp/tmp.ssx8WvfRCs /tmp/tmp.ILbukw6pEp ++ return 0 + client_pod=pxc-client-7c98b85c48-9n2xs + wait_pod pxc-client-7c98b85c48-9n2xs + local pod=pxc-client-7c98b85c48-9n2xs + local max_retry=480 + local ns= ++ echo pxc-client-7c98b85c48-9n2xs ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-7c98b85c48-9n2xs.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.hd9zbR3qhS/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.hd9zbR3qhS/select-1.sql + for i in '$(seq 0 $((cluster_size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.31-23.2 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h upgrade-haproxy-pxc-2.upgrade-haproxy-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yPGAoPqJf4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Jfoh1xiqeM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yPGAoPqJf4 ++ cat /tmp/tmp.Jfoh1xiqeM ++ rm /tmp/tmp.yPGAoPqJf4 /tmp/tmp.Jfoh1xiqeM ++ return 0 + client_pod=pxc-client-7c98b85c48-9n2xs + wait_pod pxc-client-7c98b85c48-9n2xs + local pod=pxc-client-7c98b85c48-9n2xs + local max_retry=480 + local ns= ++ echo pxc-client-7c98b85c48-9n2xs ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-7c98b85c48-9n2xs.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.hd9zbR3qhS/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1131/e2e-tests/upgrade-haproxy/compare/select-1.sql /tmp/tmp.hd9zbR3qhS/select-1.sql ++ kubectl_bin get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.URkimYA6u8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.nuNw08xmg1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod -n pxc-operator --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[*].spec.containers[?(@.name == "percona-xtradb-cluster-operator")].image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.URkimYA6u8 ++ cat /tmp/tmp.nuNw08xmg1 ++ rm /tmp/tmp.URkimYA6u8 /tmp/tmp.nuNw08xmg1 ++ return 0 + [[ perconalab/percona-xtradb-cluster-operator:PR-1131-d64e70d4 == perconalab/percona-xtradb-cluster-operator:PR-1131-d64e70d4 ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.proxysql.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kMelI939uY +++ mktemp ++ local LAST_ERR=/tmp/tmp.HYwQUpqcuK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.proxysql.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kMelI939uY ++ cat /tmp/tmp.HYwQUpqcuK ++ rm /tmp/tmp.kMelI939uY /tmp/tmp.HYwQUpqcuK ++ return 0 + [[ percona/percona-xtradb-cluster-operator:1.12.0-proxysql == percona/percona-xtradb-cluster-operator:1.12.0-proxysql ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TyzJzFNK7e +++ mktemp ++ local LAST_ERR=/tmp/tmp.HPdKXcIl3u ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.haproxy.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TyzJzFNK7e ++ cat /tmp/tmp.HPdKXcIl3u ++ rm /tmp/tmp.TyzJzFNK7e /tmp/tmp.HPdKXcIl3u ++ return 0 + [[ percona/percona-xtradb-cluster-operator:1.12.0-haproxy == percona/percona-xtradb-cluster-operator:1.12.0-haproxy ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.backup.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lTIcLFzFMr +++ mktemp ++ local LAST_ERR=/tmp/tmp.VIfZg8tYSD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.backup.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lTIcLFzFMr ++ cat /tmp/tmp.VIfZg8tYSD ++ rm /tmp/tmp.lTIcLFzFMr /tmp/tmp.VIfZg8tYSD ++ return 0 + [[ percona/percona-xtradb-cluster-operator:1.12.0-pxc8.0.31-backup == percona/percona-xtradb-cluster-operator:1.12.0-pxc8.0.31-backup ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.pmm.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HiEvejWs0n +++ mktemp ++ local LAST_ERR=/tmp/tmp.rsvRMjOzx0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.pmm.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HiEvejWs0n ++ cat /tmp/tmp.rsvRMjOzx0 ++ rm /tmp/tmp.HiEvejWs0n /tmp/tmp.rsvRMjOzx0 ++ return 0 + [[ percona/pmm-client:2.32.0 == percona/pmm-client:2.32.0 ]] ++ kubectl_bin get pxc upgrade-haproxy -o 'jsonpath={.spec.pxc.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AwMTDxbVx9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.dZLOlFc6Rj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc upgrade-haproxy -o 'jsonpath={.spec.pxc.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AwMTDxbVx9 ++ cat /tmp/tmp.dZLOlFc6Rj ++ rm /tmp/tmp.AwMTDxbVx9 /tmp/tmp.dZLOlFc6Rj ++ return 0 + [[ percona/percona-xtradb-cluster:8.0.31-23.2 == percona/percona-xtradb-cluster:8.0.31-23.2 ]] + : Operator image has been updated correctly + compare_generation 1 haproxy upgrade-haproxy + local generation=1 + local proxy=haproxy + local cluster=upgrade-haproxy + local current_generation + [[ haproxy == \h\a\p\r\o\x\y ]] + containers=(pxc haproxy) + for container in '"${containers[@]}"' ++ kubectl_bin get statefulset upgrade-haproxy-pxc -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gl5vFefwyh +++ mktemp ++ local LAST_ERR=/tmp/tmp.oY90YQVopk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get statefulset upgrade-haproxy-pxc -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gl5vFefwyh ++ cat /tmp/tmp.oY90YQVopk ++ rm /tmp/tmp.gl5vFefwyh /tmp/tmp.oY90YQVopk ++ return 0 + current_generation=1 + [[ 1 != \1 ]] + for container in '"${containers[@]}"' ++ kubectl_bin get statefulset upgrade-haproxy-haproxy -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.i2o5Kd6EJA +++ mktemp ++ local LAST_ERR=/tmp/tmp.JkByMmxR71 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get statefulset upgrade-haproxy-haproxy -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.i2o5Kd6EJA ++ cat /tmp/tmp.JkByMmxR71 ++ rm /tmp/tmp.i2o5Kd6EJA /tmp/tmp.JkByMmxR71 ++ return 0 + current_generation=1 + [[ 1 != \1 ]] + desc 'patch pxc images and upgrade' + set +o xtrace ----------------------------------------------------------------------------------- patch pxc images and upgrade ----------------------------------------------------------------------------------- + kubectl_bin patch pxc upgrade-haproxy --type=merge --patch '{ "spec": { "crVersion": "1.13.0", "pxc": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc8.0" }, "pmm": { "image": "perconalab/pmm-client:dev-latest" }, "haproxy": { "image": "perconalab/percona-xtradb-cluster-operator:main-haproxy" }, "proxysql": { "image": "perconalab/percona-xtradb-cluster-operator:main-proxysql" }, "backup": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup" } }}' ++ mktemp + local LAST_OUT=/tmp/tmp.c07xW7b7UW ++ mktemp + local LAST_ERR=/tmp/tmp.MXN9RRNp6G + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc upgrade-haproxy --type=merge --patch '{ "spec": { "crVersion": "1.13.0", "pxc": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc8.0" }, "pmm": { "image": "perconalab/pmm-client:dev-latest" }, "haproxy": { "image": "perconalab/percona-xtradb-cluster-operator:main-haproxy" }, "proxysql": { "image": "perconalab/percona-xtradb-cluster-operator:main-proxysql" }, "backup": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup" } }}' + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc upgrade-haproxy --type=merge --patch '{ "spec": { "crVersion": "1.13.0", "pxc": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc8.0" }, "pmm": { "image": "perconalab/pmm-client:dev-latest" }, "haproxy": { "image": "perconalab/percona-xtradb-cluster-operator:main-haproxy" }, "proxysql": { "image": "perconalab/percona-xtradb-cluster-operator:main-proxysql" }, "backup": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup" } }}' + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc upgrade-haproxy --type=merge --patch '{ "spec": { "crVersion": "1.13.0", "pxc": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc8.0" }, "pmm": { "image": "perconalab/pmm-client:dev-latest" }, "haproxy": { "image": "perconalab/percona-xtradb-cluster-operator:main-haproxy" }, "proxysql": { "image": "perconalab/percona-xtradb-cluster-operator:main-proxysql" }, "backup": { "image": "perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup" } }}' + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.c07xW7b7UW + cat /tmp/tmp.MXN9RRNp6G Error from server (InternalError): Internal error occurred: failed calling webhook "validationwebhook.pxc.percona.com": failed to call webhook: Post "https://percona-xtradb-cluster-operator.pxc-operator.svc:443/validate-percona-xtradbcluster?timeout=10s": no endpoints available for service "percona-xtradb-cluster-operator" + rm /tmp/tmp.c07xW7b7UW /tmp/tmp.MXN9RRNp6G + return 1