++ echo 'Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/logs/smart-update.log' Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/logs/smart-update.log ++ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP= ++ oc get projects error: No Auth Provider found for name "gcp" +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep '\-eks\-' ++ '[' ']' ++ EKS=0 +++ kubectl version -o json +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/bin/sed -r 's/[^0-9.]+//g' ++ KUBE_VERSION=1.20 +++ helm version -c +++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/' ++ HELM_VERSION=v3.8.1 ++ '[' v3 == v2 ']' + API=pxc.percona.com/v9-9-9 + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + CLUSTER=smart-update + CLUSTER_SIZE=3 + PROXY_SIZE=2 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0 ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' + PXC_VER=8.0 + TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + VS_URL=http://version-service + VS_PORT=11000 + VS_ENDPOINT=http://version-service:11000 + main + create_infra smart-update-20861 + local ns=smart-update-20861 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl patch pxc -n smart-update-21298 smart-update --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/smart-update patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.7y5WADOP1v ++ mktemp + local LAST_ERR=/tmp/tmp.umLokUdMm8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.7y5WADOP1v perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.umLokUdMm8 + rm /tmp/tmp.7y5WADOP1v /tmp/tmp.umLokUdMm8 + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.t68eSJresa ++ mktemp + local LAST_ERR=/tmp/tmp.tN48rBLkLj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.t68eSJresa No resources found + cat /tmp/tmp.tN48rBLkLj + rm /tmp/tmp.t68eSJresa /tmp/tmp.tN48rBLkLj + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.bAJQRiHQKr ++ mktemp + local LAST_ERR=/tmp/tmp.ONhQOXot0v + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.bAJQRiHQKr No resources found + cat /tmp/tmp.ONhQOXot0v + rm /tmp/tmp.bAJQRiHQKr /tmp/tmp.ONhQOXot0v + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + '[' '!' -z '' ']' + kubectl_bin delete namespace pxc-operator + xargs kubectl delete ns ++ mktemp + awk '{print$1}' + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^NAME' + kubectl_bin get ns + local LAST_OUT=/tmp/tmp.3Zh7ADYhcz ++ mktemp + local LAST_OUT=/tmp/tmp.EFRF6QSxHm ++ mktemp + local LAST_ERR=/tmp/tmp.tCW5f12lQC + local exit_status=0 ++ seq 0 2 ++ mktemp + local LAST_ERR=/tmp/tmp.X2i2eWK7zi + local exit_status=0 + for i in '$(seq 0 2)' + kubectl delete namespace pxc-operator ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl get ns + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.EFRF6QSxHm + cat /tmp/tmp.X2i2eWK7zi + rm /tmp/tmp.EFRF6QSxHm /tmp/tmp.X2i2eWK7zi + return 0 namespace "cert-manager" deleted namespace "smart-update-21298" deleted + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.3Zh7ADYhcz namespace "pxc-operator" deleted + cat /tmp/tmp.tCW5f12lQC + rm /tmp/tmp.3Zh7ADYhcz /tmp/tmp.tCW5f12lQC + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + set +o xtrace namespace/pxc-operator - Error from server (NotFound): namespaces "pxc-operator" not found + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.hCwgJms5oi ++ mktemp + local LAST_ERR=/tmp/tmp.9BSUnQngzn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl create namespace pxc-operator + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.hCwgJms5oi namespace/pxc-operator created + cat /tmp/tmp.9BSUnQngzn + rm /tmp/tmp.hCwgJms5oi /tmp/tmp.9BSUnQngzn + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.yrXmQ7TLjn +++ mktemp ++ local LAST_ERR=/tmp/tmp.8z3B2L76rd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.yrXmQ7TLjn ++ cat /tmp/tmp.8z3B2L76rd ++ rm /tmp/tmp.yrXmQ7TLjn /tmp/tmp.8z3B2L76rd ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-upgrade --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.n6Nd7w8s8y ++ mktemp + local LAST_ERR=/tmp/tmp.aFptUYSzLh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-upgrade --namespace=pxc-operator + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.n6Nd7w8s8y Context "gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-upgrade" modified. + cat /tmp/tmp.aFptUYSzLh + rm /tmp/tmp.n6Nd7w8s8y /tmp/tmp.aFptUYSzLh + return 0 + deploy_operator + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.kIW9unnrGB ++ mktemp + local LAST_ERR=/tmp/tmp.xLnlHC5NSi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/crd.yaml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.kIW9unnrGB customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com configured customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbbackups.pxc.percona.com configured + cat /tmp/tmp.xLnlHC5NSi + rm /tmp/tmp.kIW9unnrGB /tmp/tmp.xLnlHC5NSi + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/cw-rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.RjGJrByWpV ++ mktemp + local LAST_ERR=/tmp/tmp.NeJMeFPMvh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.RjGJrByWpV clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.NeJMeFPMvh + rm /tmp/tmp.RjGJrByWpV /tmp/tmp.NeJMeFPMvh + return 0 + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/cw-operator.yaml + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a^' ++ mktemp + local LAST_OUT=/tmp/tmp.04o3dvtkf7 ++ mktemp + local LAST_ERR=/tmp/tmp.GI76N6QGc0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.04o3dvtkf7 deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.GI76N6QGc0 + rm /tmp/tmp.04o3dvtkf7 /tmp/tmp.GI76N6QGc0 + return 0 + sleep 10 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.9FtKZ7KOZ5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4jJMkUvZGz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.9FtKZ7KOZ5 ++ cat /tmp/tmp.4jJMkUvZGz ++ rm /tmp/tmp.9FtKZ7KOZ5 /tmp/tmp.4jJMkUvZGz ++ return 0 + wait_pod percona-xtradb-cluster-operator-5699d7755d-lkztd 480 pxc-operator + local pod=percona-xtradb-cluster-operator-5699d7755d-lkztd + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-5699d7755d-lkztd ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace percona-xtradb-cluster-operator-5699d7755d-lkztd.Ok + sleep 3 + create_namespace smart-update-20861 + local namespace=smart-update-20861 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + kubectl_bin get ns + '[' '!' -z '' ']' ++ mktemp + kubectl_bin delete namespace smart-update-20861 + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^NAME' + awk '{print$1}' + local LAST_OUT=/tmp/tmp.XRzeeU2fW0 ++ mktemp + local LAST_ERR=/tmp/tmp.V0fkX1R7EV + local exit_status=0 + xargs kubectl delete ns ++ mktemp ++ seq 0 2 + local LAST_OUT=/tmp/tmp.BGCI92ud2E + for i in '$(seq 0 2)' + kubectl get ns ++ mktemp + local LAST_ERR=/tmp/tmp.aTCJ6kyILj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete namespace smart-update-20861 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.XRzeeU2fW0 + cat /tmp/tmp.V0fkX1R7EV + rm /tmp/tmp.XRzeeU2fW0 /tmp/tmp.V0fkX1R7EV + return 0 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace smart-update-20861 error: resource(s) were provided, but no name, label selector, or --all flag specified + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace smart-update-20861 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + cat /tmp/tmp.BGCI92ud2E + cat /tmp/tmp.aTCJ6kyILj Error from server (NotFound): namespaces "smart-update-20861" not found + rm /tmp/tmp.BGCI92ud2E /tmp/tmp.aTCJ6kyILj + return 1 + : + wait_for_delete namespace/smart-update-20861 + local res=namespace/smart-update-20861 + set +o xtrace namespace/smart-update-20861 - Error from server (NotFound): namespaces "smart-update-20861" not found + kubectl_bin create namespace smart-update-20861 ++ mktemp + local LAST_OUT=/tmp/tmp.1qOneXRPgv ++ mktemp + local LAST_ERR=/tmp/tmp.C81B18dVzL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl create namespace smart-update-20861 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.1qOneXRPgv namespace/smart-update-20861 created + cat /tmp/tmp.C81B18dVzL + rm /tmp/tmp.1qOneXRPgv /tmp/tmp.C81B18dVzL + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.mtWa2Jx9Nh +++ mktemp ++ local LAST_ERR=/tmp/tmp.ykzvgxxdtF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.mtWa2Jx9Nh ++ cat /tmp/tmp.ykzvgxxdtF ++ rm /tmp/tmp.mtWa2Jx9Nh /tmp/tmp.ykzvgxxdtF ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-upgrade --namespace=smart-update-20861 ++ mktemp + local LAST_OUT=/tmp/tmp.4FiJqBkLjv ++ mktemp + local LAST_ERR=/tmp/tmp.3wkvhmshyB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-upgrade --namespace=smart-update-20861 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.4FiJqBkLjv Context "gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-upgrade" modified. + cat /tmp/tmp.3wkvhmshyB + rm /tmp/tmp.4FiJqBkLjv /tmp/tmp.3wkvhmshyB + return 0 + apply_secrets + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.1xHeqo8TX3 ++ mktemp + local LAST_ERR=/tmp/tmp.kCbJIL957y + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.1xHeqo8TX3 secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.kCbJIL957y + rm /tmp/tmp.1xHeqo8TX3 /tmp/tmp.kCbJIL957y + return 0 + deploy_version_service + desc 'install version service' + set +o xtrace ----------------------------------------------------------------------------------- install version service ----------------------------------------------------------------------------------- + kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/operator.9.9.9.pxc-operator.json ++ mktemp + local LAST_OUT=/tmp/tmp.TpRii7ByHA ++ mktemp + local LAST_ERR=/tmp/tmp.1IWIzWCt1L + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/operator.9.9.9.pxc-operator.json + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.TpRii7ByHA configmap/versions created + cat /tmp/tmp.1IWIzWCt1L + rm /tmp/tmp.TpRii7ByHA /tmp/tmp.1IWIzWCt1L + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.V3XTrRfOGj ++ mktemp + local LAST_ERR=/tmp/tmp.3rlus1ZmBQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/vs.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.V3XTrRfOGj deployment.apps/version-service created service/version-service created + cat /tmp/tmp.3rlus1ZmBQ + rm /tmp/tmp.V3XTrRfOGj /tmp/tmp.3rlus1ZmBQ + return 0 + sleep 10 + deploy_cert_manager + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.H9V4wB2wMR ++ mktemp + local LAST_ERR=/tmp/tmp.KzRNtNYJMt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl create namespace cert-manager + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.H9V4wB2wMR namespace/cert-manager created + cat /tmp/tmp.KzRNtNYJMt + rm /tmp/tmp.H9V4wB2wMR /tmp/tmp.KzRNtNYJMt + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.8UhZKv8wfK ++ mktemp + local LAST_ERR=/tmp/tmp.tXX4uyr5an + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.8UhZKv8wfK namespace/cert-manager labeled + cat /tmp/tmp.tXX4uyr5an + rm /tmp/tmp.8UhZKv8wfK /tmp/tmp.tXX4uyr5an + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.WdCebXsyxU ++ mktemp + local LAST_ERR=/tmp/tmp.yPXVf1uxYC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml --validate=false + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.WdCebXsyxU customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged namespace/cert-manager configured serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews configured role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection configured rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.yPXVf1uxYC Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.WdCebXsyxU /tmp/tmp.yPXVf1uxYC + return 0 + sleep 60 ++ head -n1 ++ tail -n3 ++ sort -V ++ grep :8.0 ++ jq -r '.versions[].matrix.pxc[].imagePath' +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++++ grep -c percona-xtradb-cluster-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++++ mktemp +++ local LAST_OUT=/tmp/tmp.eGNIJ7URdN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vo9IunDLog +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.eGNIJ7URdN +++ cat /tmp/tmp.vo9IunDLog +++ rm /tmp/tmp.eGNIJ7URdN /tmp/tmp.vo9IunDLog +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-5699d7755d-lkztd -n pxc-operator -- curl -s http://version-service.smart-update-20861.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.eSKzFD55kr +++ mktemp ++ local LAST_ERR=/tmp/tmp.XYYgnQ4pjQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl exec -ti percona-xtradb-cluster-operator-5699d7755d-lkztd -n pxc-operator -- curl -s http://version-service.smart-update-20861.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.eSKzFD55kr ++ cat /tmp/tmp.XYYgnQ4pjQ Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.eSKzFD55kr /tmp/tmp.XYYgnQ4pjQ ++ return 0 + IMAGE_PXC=percona/percona-xtradb-cluster:8.0.19-10.1 + kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' ++ mktemp + local LAST_OUT=/tmp/tmp.s1hmTAGulT ++ mktemp + local LAST_ERR=/tmp/tmp.ZLtzqErJCE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.s1hmTAGulT customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched + cat /tmp/tmp.ZLtzqErJCE + rm /tmp/tmp.s1hmTAGulT /tmp/tmp.ZLtzqErJCE + return 0 + desc 'Updating ProxySQL PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- Updating ProxySQL PXC cluster ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/smart-update.yml /tmp/tmp.FUQtM3DI37/smart-update.yml + yq w -i /tmp/tmp.FUQtM3DI37/smart-update.yml spec.initImage perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a + spinup_pxc smart-update /tmp/tmp.FUQtM3DI37/smart-update.yml + local cluster=smart-update + local config=/tmp/tmp.FUQtM3DI37/smart-update.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.NqbWWqsy5L ++ mktemp + local LAST_ERR=/tmp/tmp.ULyhp3icew + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.NqbWWqsy5L secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.ULyhp3icew + rm /tmp/tmp.NqbWWqsy5L /tmp/tmp.ULyhp3icew + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + local LAST_OUT=/tmp/tmp.ORb3s740QZ ++ mktemp + local LAST_ERR=/tmp/tmp.9wu0Zs4UAT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update-20861~ + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.ORb3s740QZ deployment.apps/pxc-client created + cat /tmp/tmp.9wu0Zs4UAT + rm /tmp/tmp.ORb3s740QZ /tmp/tmp.9wu0Zs4UAT + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.FUQtM3DI37/smart-update.yml + '[' -z '' ']' + cat_config /tmp/tmp.FUQtM3DI37/smart-update.yml + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + local LAST_OUT=/tmp/tmp.9ZjYcxA8uM + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update-20861~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + cat /tmp/tmp.FUQtM3DI37/smart-update.yml + local LAST_ERR=/tmp/tmp.ixmcLK8Rt6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.9ZjYcxA8uM perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.ixmcLK8Rt6 + rm /tmp/tmp.9ZjYcxA8uM /tmp/tmp.ixmcLK8Rt6 + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HgQ1JLX3M9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.T2cHICisfr +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.HgQ1JLX3M9 +++ cat /tmp/tmp.T2cHICisfr +++ rm /tmp/tmp.HgQ1JLX3M9 /tmp/tmp.T2cHICisfr +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JYqnTnwSgq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.j7TBsR8pRE +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.JYqnTnwSgq +++ cat /tmp/tmp.j7TBsR8pRE +++ rm /tmp/tmp.JYqnTnwSgq /tmp/tmp.j7TBsR8pRE +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-proxysql ++ return + local proxy=smart-update-proxysql + wait_for_running smart-update-proxysql 1 + local name=smart-update-proxysql + let last_pod=0 + : + local max_retry=480 ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-proxysql-0 480 + local pod=smart-update-proxysql-0 + local max_retry=480 + local ns= ++ echo smart-update-proxysql-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace smart-update-proxysql-0..........Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-0.......................Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-1..............................Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-2...................................Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-proxysql -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-proxysql -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gW9MO1Ogyp +++ mktemp ++ local LAST_ERR=/tmp/tmp.lgfUun9bJe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.gW9MO1Ogyp ++ cat /tmp/tmp.lgfUun9bJe ++ rm /tmp/tmp.gW9MO1Ogyp /tmp/tmp.lgfUun9bJe ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-fjdhh + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-proxysql -uroot -proot_password' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-proxysql -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.txUNCs9Bxc +++ mktemp ++ local LAST_ERR=/tmp/tmp.yP80vb6JX9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.txUNCs9Bxc ++ cat /tmp/tmp.yP80vb6JX9 ++ rm /tmp/tmp.txUNCs9Bxc /tmp/tmp.yP80vb6JX9 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-fjdhh + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RDYa6elFAp +++ mktemp ++ local LAST_ERR=/tmp/tmp.Gv6ELFDtQ9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.RDYa6elFAp ++ cat /tmp/tmp.Gv6ELFDtQ9 ++ rm /tmp/tmp.RDYa6elFAp /tmp/tmp.Gv6ELFDtQ9 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.238lE2HI1I +++ mktemp ++ local LAST_ERR=/tmp/tmp.A3G2MhjmyI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.238lE2HI1I ++ cat /tmp/tmp.A3G2MhjmyI ++ rm /tmp/tmp.238lE2HI1I /tmp/tmp.A3G2MhjmyI ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-fjdhh + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EhfSsnPSmx +++ mktemp ++ local LAST_ERR=/tmp/tmp.KSfwXqB2Ka ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.EhfSsnPSmx ++ cat /tmp/tmp.KSfwXqB2Ka ++ rm /tmp/tmp.EhfSsnPSmx /tmp/tmp.KSfwXqB2Ka ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-fjdhh + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hHWjRxt8vd +++ mktemp ++ local LAST_ERR=/tmp/tmp.r9dQHDSot4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.hHWjRxt8vd ++ cat /tmp/tmp.r9dQHDSot4 Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.hHWjRxt8vd /tmp/tmp.r9dQHDSot4 ++ return 0 + '[' '' ']' +++ get_proxy smart-update +++ local target_cluster=smart-update ++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.NtWps5tmnu +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.5P2C3IMdZ0 ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ cat /tmp/tmp.NtWps5tmnu ++++ cat /tmp/tmp.5P2C3IMdZ0 ++++ rm /tmp/tmp.NtWps5tmnu /tmp/tmp.5P2C3IMdZ0 ++++ return 0 +++ [[ '' == \t\r\u\e ]] ++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.k4ZphgAuoQ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.bCH1PEA46Q ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ cat /tmp/tmp.k4ZphgAuoQ ++++ cat /tmp/tmp.bCH1PEA46Q ++++ rm /tmp/tmp.k4ZphgAuoQ /tmp/tmp.bCH1PEA46Q ++++ return 0 +++ [[ true == \t\r\u\e ]] +++ echo smart-update-proxysql +++ return ++ get_proxy_primary '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=smart-update-proxysql-0 +++ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status="ONLINE";' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 proxysql +++ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status="ONLINE";' +++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' +++ local pod=smart-update-proxysql-0 +++ local container_name=proxysql +++ set +o xtrace ++ local ip=smart-update-pxc-0.smart-update-pxc.smart-update-20861.svc.cluster.local +++ echo smart-update-pxc-0.smart-update-pxc.smart-update-20861.svc.cluster.local +++ wc -l ++ '[' 1 '!=' 1 ']' ++ echo smart-update-pxc-0.smart-update-pxc.smart-update-20861.svc.cluster.local ++ cut -d. -f1 + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.HxWstZb6k8 ++ mktemp + local LAST_ERR=/tmp/tmp.a3McFlwCu1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.HxWstZb6k8 perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.a3McFlwCu1 + rm /tmp/tmp.HxWstZb6k8 /tmp/tmp.a3McFlwCu1 + return 0 + sleep 7 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update.................................................... smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U0xTalM87m +++ mktemp ++ local LAST_ERR=/tmp/tmp.dcW2zis9gy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.U0xTalM87m ++ cat /tmp/tmp.dcW2zis9gy ++ rm /tmp/tmp.U0xTalM87m /tmp/tmp.dcW2zis9gy ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4qX1pCngle +++ mktemp ++ local LAST_ERR=/tmp/tmp.q9b1dC3FT4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.4qX1pCngle ++ cat /tmp/tmp.q9b1dC3FT4 ++ rm /tmp/tmp.4qX1pCngle /tmp/tmp.q9b1dC3FT4 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Qa6YSR2PpC +++ mktemp ++ local LAST_ERR=/tmp/tmp.1fhZvt1pRm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Qa6YSR2PpC ++ cat /tmp/tmp.1fhZvt1pRm ++ rm /tmp/tmp.Qa6YSR2PpC /tmp/tmp.1fhZvt1pRm ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oV3uqotdxx +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZhpJqtG3yJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.oV3uqotdxx ++ cat /tmp/tmp.ZhpJqtG3yJ ++ rm /tmp/tmp.oV3uqotdxx /tmp/tmp.ZhpJqtG3yJ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vl3omXAKQc +++ mktemp ++ local LAST_ERR=/tmp/tmp.lafys9zqqo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.vl3omXAKQc ++ cat /tmp/tmp.lafys9zqqo ++ rm /tmp/tmp.vl3omXAKQc /tmp/tmp.lafys9zqqo ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iG6ZlCZDl3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FkCq1LWTqf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.iG6ZlCZDl3 ++ cat /tmp/tmp.FkCq1LWTqf ++ rm /tmp/tmp.iG6ZlCZDl3 /tmp/tmp.FkCq1LWTqf ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GnnVAdluWz +++ mktemp ++ local LAST_ERR=/tmp/tmp.dvlHuUnW7u ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.GnnVAdluWz ++ cat /tmp/tmp.dvlHuUnW7u ++ rm /tmp/tmp.GnnVAdluWz /tmp/tmp.dvlHuUnW7u ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DRYLadRSS1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.xpFIOtDUDf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.DRYLadRSS1 ++ cat /tmp/tmp.xpFIOtDUDf ++ rm /tmp/tmp.DRYLadRSS1 /tmp/tmp.xpFIOtDUDf ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9hme5qoUyI +++ mktemp ++ local LAST_ERR=/tmp/tmp.9sO9fx3yfQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.9hme5qoUyI ++ cat /tmp/tmp.9sO9fx3yfQ ++ rm /tmp/tmp.9hme5qoUyI /tmp/tmp.9sO9fx3yfQ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BSbN6JRwkg +++ mktemp ++ local LAST_ERR=/tmp/tmp.Pik8uYAegz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.BSbN6JRwkg ++ cat /tmp/tmp.Pik8uYAegz ++ rm /tmp/tmp.BSbN6JRwkg /tmp/tmp.Pik8uYAegz ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XBDomNRYKj +++ mktemp ++ local LAST_ERR=/tmp/tmp.cXQSKRHhpU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.XBDomNRYKj ++ cat /tmp/tmp.cXQSKRHhpU ++ rm /tmp/tmp.XBDomNRYKj /tmp/tmp.cXQSKRHhpU ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8ja4wAcYh4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.N35lxWO6lF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.8ja4wAcYh4 ++ cat /tmp/tmp.N35lxWO6lF ++ rm /tmp/tmp.8ja4wAcYh4 /tmp/tmp.N35lxWO6lF ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ww7MhGRoVt +++ mktemp ++ local LAST_ERR=/tmp/tmp.AvHct4w3De ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.ww7MhGRoVt ++ cat /tmp/tmp.AvHct4w3De ++ rm /tmp/tmp.ww7MhGRoVt /tmp/tmp.AvHct4w3De ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MwdEQY4Dvo +++ mktemp ++ local LAST_ERR=/tmp/tmp.DbF1YfAOxw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.MwdEQY4Dvo ++ cat /tmp/tmp.DbF1YfAOxw ++ rm /tmp/tmp.MwdEQY4Dvo /tmp/tmp.DbF1YfAOxw ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZINTyFDDW5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.BpjJYW1yc8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.ZINTyFDDW5 ++ cat /tmp/tmp.BpjJYW1yc8 ++ rm /tmp/tmp.ZINTyFDDW5 /tmp/tmp.BpjJYW1yc8 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.R12Etyy6LL +++ mktemp ++ local LAST_ERR=/tmp/tmp.bBTbiTHbl4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.R12Etyy6LL ++ cat /tmp/tmp.bBTbiTHbl4 ++ rm /tmp/tmp.R12Etyy6LL /tmp/tmp.bBTbiTHbl4 ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PfC6cGEn4S +++ mktemp ++ local LAST_ERR=/tmp/tmp.HPy60Wor5y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.PfC6cGEn4S ++ cat /tmp/tmp.HPy60Wor5y ++ rm /tmp/tmp.PfC6cGEn4S /tmp/tmp.HPy60Wor5y ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.14llBhmnNe ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.RPR74EUZMR +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ cat /tmp/tmp.14llBhmnNe +++++ cat /tmp/tmp.RPR74EUZMR +++++ rm /tmp/tmp.14llBhmnNe /tmp/tmp.RPR74EUZMR +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.a77GFeIgUy ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ZZypiy6m45 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ cat /tmp/tmp.a77GFeIgUy +++++ cat /tmp/tmp.ZZypiy6m45 +++++ rm /tmp/tmp.a77GFeIgUy /tmp/tmp.ZZypiy6m45 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-proxysql ++++ return +++ local cluster_proxy=smart-update-proxysql +++ echo proxysql ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mef7Do6Md3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.X2sVsUpIFb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.mef7Do6Md3 ++ cat /tmp/tmp.X2sVsUpIFb ++ rm /tmp/tmp.mef7Do6Md3 /tmp/tmp.X2sVsUpIFb ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aGb0KMWZYW +++ mktemp ++ local LAST_ERR=/tmp/tmp.MxkSLEZxPS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.aGb0KMWZYW ++ cat /tmp/tmp.MxkSLEZxPS ++ rm /tmp/tmp.aGb0KMWZYW /tmp/tmp.MxkSLEZxPS ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DjKe2CncDb +++ mktemp ++ local LAST_ERR=/tmp/tmp.DHkkBlETR4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.DjKe2CncDb ++ cat /tmp/tmp.DHkkBlETR4 ++ rm /tmp/tmp.DjKe2CncDb /tmp/tmp.DHkkBlETR4 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8PZPymUbDR +++ mktemp ++ local LAST_ERR=/tmp/tmp.x6ZKpWR2gL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.8PZPymUbDR ++ cat /tmp/tmp.x6ZKpWR2gL ++ rm /tmp/tmp.8PZPymUbDR /tmp/tmp.x6ZKpWR2gL ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/smart-update.yml ++ mktemp + local LAST_OUT=/tmp/tmp.CipbDzDmFk ++ mktemp + local LAST_ERR=/tmp/tmp.Vw7NvqUOoC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/smart-update.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.CipbDzDmFk perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.Vw7NvqUOoC + rm /tmp/tmp.CipbDzDmFk /tmp/tmp.Vw7NvqUOoC + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.NYQ2NiWuR2 ++ mktemp + local LAST_ERR=/tmp/tmp.LtC5876bTy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pvc --all + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.NYQ2NiWuR2 persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted persistentvolumeclaim "proxydata-smart-update-proxysql-0" deleted persistentvolumeclaim "proxydata-smart-update-proxysql-1" deleted + cat /tmp/tmp.LtC5876bTy + rm /tmp/tmp.NYQ2NiWuR2 /tmp/tmp.LtC5876bTy + return 0 + desc 'Updating HAProxy PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- Updating HAProxy PXC cluster ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/smart-update-haproxy.yml /tmp/tmp.FUQtM3DI37/smart-update-haproxy.yml + yq w -i /tmp/tmp.FUQtM3DI37/smart-update-haproxy.yml spec.initImage perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a + spinup_pxc smart-update /tmp/tmp.FUQtM3DI37/smart-update-haproxy.yml + local cluster=smart-update + local config=/tmp/tmp.FUQtM3DI37/smart-update-haproxy.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.qDu7NkbEAe ++ mktemp + local LAST_ERR=/tmp/tmp.rzVAaxJj18 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.qDu7NkbEAe secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.rzVAaxJj18 + rm /tmp/tmp.qDu7NkbEAe /tmp/tmp.rzVAaxJj18 + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.WagWnm1jF0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update-20861~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ mktemp + local LAST_ERR=/tmp/tmp.wsKBUZFwUN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.WagWnm1jF0 deployment.apps/pxc-client unchanged + cat /tmp/tmp.wsKBUZFwUN + rm /tmp/tmp.WagWnm1jF0 /tmp/tmp.wsKBUZFwUN + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.FUQtM3DI37/smart-update-haproxy.yml + '[' -z '' ']' + cat_config /tmp/tmp.FUQtM3DI37/smart-update-haproxy.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.W2pwLsSd4H + cat /tmp/tmp.FUQtM3DI37/smart-update-haproxy.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.N7tt5hOW0W + local exit_status=0 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update-20861~ + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + for i in '$(seq 0 2)' + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.W2pwLsSd4H perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.N7tt5hOW0W + rm /tmp/tmp.W2pwLsSd4H /tmp/tmp.N7tt5hOW0W + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Glahy7DQ0T ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hB1JDxCNdR +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.Glahy7DQ0T +++ cat /tmp/tmp.hB1JDxCNdR +++ rm /tmp/tmp.Glahy7DQ0T /tmp/tmp.hB1JDxCNdR +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-haproxy-0 + local container= + set +o xtrace smart-update-haproxy-0.......................................error: a container name must be specified for pod smart-update-haproxy-0, choose one of: [haproxy pxc-monit] .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-1..................................Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-2...............................Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZHXA3pbScT +++ mktemp ++ local LAST_ERR=/tmp/tmp.hH52NYrxU6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.ZHXA3pbScT ++ cat /tmp/tmp.hH52NYrxU6 ++ rm /tmp/tmp.ZHXA3pbScT /tmp/tmp.hH52NYrxU6 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.o56ldBYXyT +++ mktemp ++ local LAST_ERR=/tmp/tmp.jTbieTNJN7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.o56ldBYXyT ++ cat /tmp/tmp.jTbieTNJN7 ++ rm /tmp/tmp.o56ldBYXyT /tmp/tmp.jTbieTNJN7 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JsNecdJ0U9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4oy9buCWJ6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.JsNecdJ0U9 ++ cat /tmp/tmp.4oy9buCWJ6 ++ rm /tmp/tmp.JsNecdJ0U9 /tmp/tmp.4oy9buCWJ6 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w0uiDImGQb +++ mktemp ++ local LAST_ERR=/tmp/tmp.6tsxPNRy8S ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.w0uiDImGQb ++ cat /tmp/tmp.6tsxPNRy8S ++ rm /tmp/tmp.w0uiDImGQb /tmp/tmp.6tsxPNRy8S ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sFI99Tpu1I +++ mktemp ++ local LAST_ERR=/tmp/tmp.4PL8H5J6Bl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.sFI99Tpu1I ++ cat /tmp/tmp.4PL8H5J6Bl ++ rm /tmp/tmp.sFI99Tpu1I /tmp/tmp.4PL8H5J6Bl ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-fjdhh + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.004UlrfMW3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ojwgB5teX2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.004UlrfMW3 ++ cat /tmp/tmp.ojwgB5teX2 Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.004UlrfMW3 /tmp/tmp.ojwgB5teX2 ++ return 0 + '[' '' ']' ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jYuupljtOV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qjqAgbTrj1 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.jYuupljtOV +++ cat /tmp/tmp.qjqAgbTrj1 +++ rm /tmp/tmp.jYuupljtOV /tmp/tmp.qjqAgbTrj1 +++ return 0 ++ client_pod=pxc-client-5d749ff8b6-fjdhh ++ wait_pod pxc-client-5d749ff8b6-fjdhh ++ local pod=pxc-client-5d749ff8b6-fjdhh ++ local max_retry=480 ++ local ns= +++ echo pxc-client-5d749ff8b6-fjdhh +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.FpNPh2V7B0 ++ mktemp + local LAST_ERR=/tmp/tmp.mm4lpJ8RmE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.FpNPh2V7B0 perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.mm4lpJ8RmE + rm /tmp/tmp.FpNPh2V7B0 /tmp/tmp.mm4lpJ8RmE + return 0 + sleep 7 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update................................. smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yaoivXzwAX +++ mktemp ++ local LAST_ERR=/tmp/tmp.lx44lbIl9s ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.yaoivXzwAX ++ cat /tmp/tmp.lx44lbIl9s ++ rm /tmp/tmp.yaoivXzwAX /tmp/tmp.lx44lbIl9s ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.b0W1hTsXlv +++ mktemp ++ local LAST_ERR=/tmp/tmp.dZcvhUG3ca ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.b0W1hTsXlv ++ cat /tmp/tmp.dZcvhUG3ca ++ rm /tmp/tmp.b0W1hTsXlv /tmp/tmp.dZcvhUG3ca ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dRjKvosUlO +++ mktemp ++ local LAST_ERR=/tmp/tmp.mgL9O1aFgt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.dRjKvosUlO ++ cat /tmp/tmp.mgL9O1aFgt ++ rm /tmp/tmp.dRjKvosUlO /tmp/tmp.mgL9O1aFgt ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WD6wNNL6gX +++ mktemp ++ local LAST_ERR=/tmp/tmp.3crzQQFQ21 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.WD6wNNL6gX ++ cat /tmp/tmp.3crzQQFQ21 ++ rm /tmp/tmp.WD6wNNL6gX /tmp/tmp.3crzQQFQ21 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PF5Tts0s21 +++ mktemp ++ local LAST_ERR=/tmp/tmp.BlvsIKtpKm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.PF5Tts0s21 ++ cat /tmp/tmp.BlvsIKtpKm ++ rm /tmp/tmp.PF5Tts0s21 /tmp/tmp.BlvsIKtpKm ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hCa9SwScXa +++ mktemp ++ local LAST_ERR=/tmp/tmp.do2GNA4Bt2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.hCa9SwScXa ++ cat /tmp/tmp.do2GNA4Bt2 ++ rm /tmp/tmp.hCa9SwScXa /tmp/tmp.do2GNA4Bt2 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2AP2M1Uwlv +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZwpFUIBTaZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.2AP2M1Uwlv ++ cat /tmp/tmp.ZwpFUIBTaZ ++ rm /tmp/tmp.2AP2M1Uwlv /tmp/tmp.ZwpFUIBTaZ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K2i6S24Eiz +++ mktemp ++ local LAST_ERR=/tmp/tmp.ef6O8rOFmf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.K2i6S24Eiz ++ cat /tmp/tmp.ef6O8rOFmf ++ rm /tmp/tmp.K2i6S24Eiz /tmp/tmp.ef6O8rOFmf ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pqx0PQcB15 +++ mktemp ++ local LAST_ERR=/tmp/tmp.lypPdRIpPu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Pqx0PQcB15 ++ cat /tmp/tmp.lypPdRIpPu ++ rm /tmp/tmp.Pqx0PQcB15 /tmp/tmp.lypPdRIpPu ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.B5SEvzUnlS +++ mktemp ++ local LAST_ERR=/tmp/tmp.6wmaYx8AOQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.B5SEvzUnlS ++ cat /tmp/tmp.6wmaYx8AOQ ++ rm /tmp/tmp.B5SEvzUnlS /tmp/tmp.6wmaYx8AOQ ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ubN6per2UN +++ mktemp ++ local LAST_ERR=/tmp/tmp.WfjMrLQk6Q ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.ubN6per2UN ++ cat /tmp/tmp.WfjMrLQk6Q ++ rm /tmp/tmp.ubN6per2UN /tmp/tmp.WfjMrLQk6Q ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.IIct9fXLYY ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.BiHOyq6sSv +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ cat /tmp/tmp.IIct9fXLYY +++++ cat /tmp/tmp.BiHOyq6sSv +++++ rm /tmp/tmp.IIct9fXLYY /tmp/tmp.BiHOyq6sSv +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WmdpnohckG +++ mktemp ++ local LAST_ERR=/tmp/tmp.8HzYWMJkwG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.WmdpnohckG ++ cat /tmp/tmp.8HzYWMJkwG ++ rm /tmp/tmp.WmdpnohckG /tmp/tmp.8HzYWMJkwG ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ke5zWwRY8m +++ mktemp ++ local LAST_ERR=/tmp/tmp.XzBQV6Rwu5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Ke5zWwRY8m ++ cat /tmp/tmp.XzBQV6Rwu5 ++ rm /tmp/tmp.Ke5zWwRY8m /tmp/tmp.XzBQV6Rwu5 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.csyQCYy5ni +++ mktemp ++ local LAST_ERR=/tmp/tmp.tWRLso9J4T ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.csyQCYy5ni ++ cat /tmp/tmp.tWRLso9J4T ++ rm /tmp/tmp.csyQCYy5ni /tmp/tmp.tWRLso9J4T ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BiMba1PMER +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ml5sqlOt13 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.BiMba1PMER ++ cat /tmp/tmp.Ml5sqlOt13 ++ rm /tmp/tmp.BiMba1PMER /tmp/tmp.Ml5sqlOt13 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + kubectl_bin delete -f /tmp/tmp.FUQtM3DI37/smart-update-haproxy.yml ++ mktemp + local LAST_OUT=/tmp/tmp.JzUgQjdx6W ++ mktemp + local LAST_ERR=/tmp/tmp.lfzock3lt1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete -f /tmp/tmp.FUQtM3DI37/smart-update-haproxy.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.JzUgQjdx6W perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.lfzock3lt1 + rm /tmp/tmp.JzUgQjdx6W /tmp/tmp.lfzock3lt1 + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.pCs2fv56vQ ++ mktemp + local LAST_ERR=/tmp/tmp.vXD0AenK4W + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pvc --all + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.pCs2fv56vQ persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.vXD0AenK4W + rm /tmp/tmp.pCs2fv56vQ /tmp/tmp.vXD0AenK4W + return 0 + desc 'Updating PXC cluster with version service available but disabled' + set +o xtrace ----------------------------------------------------------------------------------- Updating PXC cluster with version service available but disabled ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/smart-update-version-service-reachable.yml /tmp/tmp.FUQtM3DI37/smart-update-version-service-reachable.yml + yq w -i /tmp/tmp.FUQtM3DI37/smart-update-version-service-reachable.yml spec.initImage perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a + spinup_pxc smart-update /tmp/tmp.FUQtM3DI37/smart-update-version-service-reachable.yml + local cluster=smart-update + local config=/tmp/tmp.FUQtM3DI37/smart-update-version-service-reachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.vslWIEOxtx ++ mktemp + local LAST_ERR=/tmp/tmp.fM4l9ytFRU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.vslWIEOxtx secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.fM4l9ytFRU + rm /tmp/tmp.vslWIEOxtx /tmp/tmp.fM4l9ytFRU + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + kubectl_bin apply -f - ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + local LAST_OUT=/tmp/tmp.cT5Mkvp4NC + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_ERR=/tmp/tmp.SVVLoVrpFU + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update-20861~ + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.cT5Mkvp4NC deployment.apps/pxc-client unchanged + cat /tmp/tmp.SVVLoVrpFU + rm /tmp/tmp.cT5Mkvp4NC /tmp/tmp.SVVLoVrpFU + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.FUQtM3DI37/smart-update-version-service-reachable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.FUQtM3DI37/smart-update-version-service-reachable.yml + cat /tmp/tmp.FUQtM3DI37/smart-update-version-service-reachable.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_OUT=/tmp/tmp.iz3H0JJebX + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update-20861~ + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.kBpD4oHcZk + local exit_status=0 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.iz3H0JJebX perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.kBpD4oHcZk + rm /tmp/tmp.iz3H0JJebX /tmp/tmp.kBpD4oHcZk + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vVCMb6Bidn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.v6KYLewzcj +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.vVCMb6Bidn +++ cat /tmp/tmp.v6KYLewzcj +++ rm /tmp/tmp.vVCMb6Bidn /tmp/tmp.v6KYLewzcj +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace smart-update-haproxy-0...................................error: a container name must be specified for pod smart-update-haproxy-0, choose one of: [haproxy pxc-monit] .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-1..........................Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace smart-update-pxc-2...................................Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Xnhpk4qvgc +++ mktemp ++ local LAST_ERR=/tmp/tmp.1PbBhtBltv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Xnhpk4qvgc ++ cat /tmp/tmp.1PbBhtBltv ++ rm /tmp/tmp.Xnhpk4qvgc /tmp/tmp.1PbBhtBltv ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.A2hYxiA9lD +++ mktemp ++ local LAST_ERR=/tmp/tmp.7W2y1HG6Pv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.A2hYxiA9lD ++ cat /tmp/tmp.7W2y1HG6Pv ++ rm /tmp/tmp.A2hYxiA9lD /tmp/tmp.7W2y1HG6Pv ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.E0cyLIEnzD +++ mktemp ++ local LAST_ERR=/tmp/tmp.ttWYF8swR3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.E0cyLIEnzD ++ cat /tmp/tmp.ttWYF8swR3 ++ rm /tmp/tmp.E0cyLIEnzD /tmp/tmp.ttWYF8swR3 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zxPc76oMta +++ mktemp ++ local LAST_ERR=/tmp/tmp.h4knn7LHHc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.zxPc76oMta ++ cat /tmp/tmp.h4knn7LHHc ++ rm /tmp/tmp.zxPc76oMta /tmp/tmp.h4knn7LHHc ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lafyn5lQvE +++ mktemp ++ local LAST_ERR=/tmp/tmp.prIurl8uSP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.lafyn5lQvE ++ cat /tmp/tmp.prIurl8uSP ++ rm /tmp/tmp.lafyn5lQvE /tmp/tmp.prIurl8uSP ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' +++ mktemp ++ egrep -o 'early-plugin-load=keyring_\w+.so' ++ local LAST_OUT=/tmp/tmp.2wZB4h989J +++ mktemp ++ local LAST_ERR=/tmp/tmp.pvinObqO1g ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.2wZB4h989J ++ cat /tmp/tmp.pvinObqO1g Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.2wZB4h989J /tmp/tmp.pvinObqO1g ++ return 0 + '[' '' ']' ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.AIimQIU74w ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kM2tAE8bEg +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.AIimQIU74w +++ cat /tmp/tmp.kM2tAE8bEg +++ rm /tmp/tmp.AIimQIU74w /tmp/tmp.kM2tAE8bEg +++ return 0 ++ client_pod=pxc-client-5d749ff8b6-fjdhh ++ wait_pod pxc-client-5d749ff8b6-fjdhh ++ local pod=pxc-client-5d749ff8b6-fjdhh ++ local max_retry=480 ++ local ns= +++ echo pxc-client-5d749ff8b6-fjdhh +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.DaYF0D4LKf ++ mktemp + local LAST_ERR=/tmp/tmp.KmD43K6grW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.DaYF0D4LKf perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.KmD43K6grW + rm /tmp/tmp.DaYF0D4LKf /tmp/tmp.KmD43K6grW + return 0 + sleep 7 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update................................ smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hgArhKN9N7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.y9aLD2mgOu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.hgArhKN9N7 ++ cat /tmp/tmp.y9aLD2mgOu ++ rm /tmp/tmp.hgArhKN9N7 /tmp/tmp.y9aLD2mgOu ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MZIB6H2KtZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.JeJIK0Kkzp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.MZIB6H2KtZ ++ cat /tmp/tmp.JeJIK0Kkzp ++ rm /tmp/tmp.MZIB6H2KtZ /tmp/tmp.JeJIK0Kkzp ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.serOveT5be +++ mktemp ++ local LAST_ERR=/tmp/tmp.IwM01wsfQr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.serOveT5be ++ cat /tmp/tmp.IwM01wsfQr ++ rm /tmp/tmp.serOveT5be /tmp/tmp.IwM01wsfQr ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rPf5k5zliP +++ mktemp ++ local LAST_ERR=/tmp/tmp.Qg8uIR4CXh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.rPf5k5zliP ++ cat /tmp/tmp.Qg8uIR4CXh ++ rm /tmp/tmp.rPf5k5zliP /tmp/tmp.Qg8uIR4CXh ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IChU6SLZfp +++ mktemp ++ local LAST_ERR=/tmp/tmp.xPRjJGaey6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.IChU6SLZfp ++ cat /tmp/tmp.xPRjJGaey6 ++ rm /tmp/tmp.IChU6SLZfp /tmp/tmp.xPRjJGaey6 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UlqxnENlgA +++ mktemp ++ local LAST_ERR=/tmp/tmp.AzbivN9gna ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.UlqxnENlgA ++ cat /tmp/tmp.AzbivN9gna ++ rm /tmp/tmp.UlqxnENlgA /tmp/tmp.AzbivN9gna ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V47kMUoE5t +++ mktemp ++ local LAST_ERR=/tmp/tmp.67SFE74X40 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.V47kMUoE5t ++ cat /tmp/tmp.67SFE74X40 ++ rm /tmp/tmp.V47kMUoE5t /tmp/tmp.67SFE74X40 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ci4bDnZCRP +++ mktemp ++ local LAST_ERR=/tmp/tmp.oiBB9isqpZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.ci4bDnZCRP ++ cat /tmp/tmp.oiBB9isqpZ ++ rm /tmp/tmp.ci4bDnZCRP /tmp/tmp.oiBB9isqpZ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7utsRHWFeo +++ mktemp ++ local LAST_ERR=/tmp/tmp.wjeuMjnjv2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.7utsRHWFeo ++ cat /tmp/tmp.wjeuMjnjv2 ++ rm /tmp/tmp.7utsRHWFeo /tmp/tmp.wjeuMjnjv2 ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UxNT5HuEGN +++ mktemp ++ local LAST_ERR=/tmp/tmp.Lt8ArFLuO4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.UxNT5HuEGN ++ cat /tmp/tmp.Lt8ArFLuO4 ++ rm /tmp/tmp.UxNT5HuEGN /tmp/tmp.Lt8ArFLuO4 ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.YKksaDhano ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.xK2xClK7ou +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ cat /tmp/tmp.YKksaDhano +++++ cat /tmp/tmp.xK2xClK7ou +++++ rm /tmp/tmp.YKksaDhano /tmp/tmp.xK2xClK7ou +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Jr2PqcYiYZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.RI12YmaJZE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Jr2PqcYiYZ ++ cat /tmp/tmp.RI12YmaJZE ++ rm /tmp/tmp.Jr2PqcYiYZ /tmp/tmp.RI12YmaJZE ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.j1mMkUTPN1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.I7cYr5hTCc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.j1mMkUTPN1 ++ cat /tmp/tmp.I7cYr5hTCc ++ rm /tmp/tmp.j1mMkUTPN1 /tmp/tmp.I7cYr5hTCc ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-fjdhh ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VMAbRvCNpU +++ mktemp ++ local LAST_ERR=/tmp/tmp.Hi5D8n04oA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.VMAbRvCNpU ++ cat /tmp/tmp.Hi5D8n04oA ++ rm /tmp/tmp.VMAbRvCNpU /tmp/tmp.Hi5D8n04oA ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.50dpovQYIn +++ mktemp ++ local LAST_ERR=/tmp/tmp.rac1NtZHti ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.50dpovQYIn ++ cat /tmp/tmp.rac1NtZHti ++ rm /tmp/tmp.50dpovQYIn /tmp/tmp.rac1NtZHti ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + kubectl_bin delete -f /tmp/tmp.FUQtM3DI37/smart-update-version-service-reachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.8ch7kEUn8G ++ mktemp + local LAST_ERR=/tmp/tmp.ugQdJ9LBkS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete -f /tmp/tmp.FUQtM3DI37/smart-update-version-service-reachable.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.8ch7kEUn8G perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.ugQdJ9LBkS + rm /tmp/tmp.8ch7kEUn8G /tmp/tmp.ugQdJ9LBkS + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.uFDDqGHkyH ++ mktemp + local LAST_ERR=/tmp/tmp.B0N9dOKcAp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pvc --all + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.uFDDqGHkyH persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.B0N9dOKcAp + rm /tmp/tmp.uFDDqGHkyH /tmp/tmp.B0N9dOKcAp + return 0 + desc 'PXC cluster with version service offline' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster with version service offline ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/smart-update-version-service-unreachable.yml /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + yq w -i /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml spec.initImage perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a + spinup_pxc smart-update /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.YQPmt2lbAe ++ mktemp + local LAST_ERR=/tmp/tmp.iqx3p2MRKJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.YQPmt2lbAe secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.iqx3p2MRKJ + rm /tmp/tmp.YQPmt2lbAe /tmp/tmp.iqx3p2MRKJ + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + local LAST_OUT=/tmp/tmp.IRuACcODK1 + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update-20861~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' ++ mktemp + local LAST_ERR=/tmp/tmp.YDfYfL1jep + local exit_status=0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.IRuACcODK1 deployment.apps/pxc-client unchanged + cat /tmp/tmp.YDfYfL1jep + rm /tmp/tmp.IRuACcODK1 /tmp/tmp.YDfYfL1jep + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + '[' -z '' ']' + cat_config /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + kubectl_bin apply -f - ++ mktemp + cat /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + local LAST_OUT=/tmp/tmp.rhKu6LdlZz + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update-20861~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_ERR=/tmp/tmp.n0uZ9uTUHX + local exit_status=0 + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.rhKu6LdlZz perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.n0uZ9uTUHX + rm /tmp/tmp.rhKu6LdlZz /tmp/tmp.n0uZ9uTUHX + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UaxIKsQt1t ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WRKNMQBY1H +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.UaxIKsQt1t +++ cat /tmp/tmp.WRKNMQBY1H +++ rm /tmp/tmp.UaxIKsQt1t /tmp/tmp.WRKNMQBY1H +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace smart-update-haproxy-0..........................................error: a container name must be specified for pod smart-update-haproxy-0, choose one of: [haproxy pxc-monit] .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-1........................Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-2 ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-2.................................Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ytcRd8kFId +++ mktemp ++ local LAST_ERR=/tmp/tmp.mOHuHiEAd1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.ytcRd8kFId ++ cat /tmp/tmp.mOHuHiEAd1 ++ rm /tmp/tmp.ytcRd8kFId /tmp/tmp.mOHuHiEAd1 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CbA9ZkTSvc +++ mktemp ++ local LAST_ERR=/tmp/tmp.4VY2sUGIQb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.CbA9ZkTSvc ++ cat /tmp/tmp.4VY2sUGIQb ++ rm /tmp/tmp.CbA9ZkTSvc /tmp/tmp.4VY2sUGIQb ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tEgBTusexp +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ai1jHgQVux ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.tEgBTusexp ++ cat /tmp/tmp.Ai1jHgQVux ++ rm /tmp/tmp.tEgBTusexp /tmp/tmp.Ai1jHgQVux ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-5d749ff8b6-fjdhh + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PCjc0R4c5n +++ mktemp ++ local LAST_ERR=/tmp/tmp.sDc9EVMcu7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.PCjc0R4c5n ++ cat /tmp/tmp.sDc9EVMcu7 ++ rm /tmp/tmp.PCjc0R4c5n /tmp/tmp.sDc9EVMcu7 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-fjdhh + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oJEt7dbEQ6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ew00Z4uMgA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.oJEt7dbEQ6 ++ cat /tmp/tmp.ew00Z4uMgA ++ rm /tmp/tmp.oJEt7dbEQ6 /tmp/tmp.ew00Z4uMgA ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ egrep -o 'early-plugin-load=keyring_\w+.so' ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ay1yBl8mlH +++ mktemp ++ local LAST_ERR=/tmp/tmp.5Z3YMiJ1q5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Ay1yBl8mlH ++ cat /tmp/tmp.5Z3YMiJ1q5 Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.Ay1yBl8mlH /tmp/tmp.5Z3YMiJ1q5 ++ return 0 + '[' '' ']' + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.W5DZhwZDfS +++ mktemp ++ local LAST_ERR=/tmp/tmp.zIU9SpOQMH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.W5DZhwZDfS ++ cat /tmp/tmp.zIU9SpOQMH ++ rm /tmp/tmp.W5DZhwZDfS /tmp/tmp.zIU9SpOQMH ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lNZQR26iVf +++ mktemp ++ local LAST_ERR=/tmp/tmp.upKh5AI16z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.lNZQR26iVf ++ cat /tmp/tmp.upKh5AI16z ++ rm /tmp/tmp.lNZQR26iVf /tmp/tmp.upKh5AI16z ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.4h6ZPQNGhX ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.VJ2xTad98E +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ cat /tmp/tmp.4h6ZPQNGhX +++++ cat /tmp/tmp.VJ2xTad98E +++++ rm /tmp/tmp.4h6ZPQNGhX /tmp/tmp.VJ2xTad98E +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1sJgT0jp0a +++ mktemp ++ local LAST_ERR=/tmp/tmp.3lVj6Upw6y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.1sJgT0jp0a ++ cat /tmp/tmp.3lVj6Upw6y ++ rm /tmp/tmp.1sJgT0jp0a /tmp/tmp.3lVj6Upw6y ++ return 0 + [[ 2 == \2 ]] ++ kubectl_bin get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.k7K8ePnL3a +++ mktemp ++ local LAST_ERR=/tmp/tmp.VYYZaOK6Xi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.k7K8ePnL3a ++ cat /tmp/tmp.VYYZaOK6Xi ++ rm /tmp/tmp.k7K8ePnL3a /tmp/tmp.VYYZaOK6Xi ++ return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 != \p\e\r\c\o\n\a\/\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\:\8\.\0\.\1\9\-\1\0\.\1 ]] + desc 'PXC cluster update with recommended image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with recommended image by version service ----------------------------------------------------------------------------------- + vs_image=recommended ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tYoUUM6dsS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.IedlMEYoPW +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.tYoUUM6dsS +++ cat /tmp/tmp.IedlMEYoPW +++ rm /tmp/tmp.tYoUUM6dsS /tmp/tmp.IedlMEYoPW +++ return 0 ++ client_pod=pxc-client-5d749ff8b6-fjdhh ++ wait_pod pxc-client-5d749ff8b6-fjdhh ++ local pod=pxc-client-5d749ff8b6-fjdhh ++ local max_retry=480 ++ local ns= +++ echo pxc-client-5d749ff8b6-fjdhh +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.JcCQ9RsBN5 ++ mktemp + local LAST_ERR=/tmp/tmp.QqpNenGEc7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.JcCQ9RsBN5 perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.QqpNenGEc7 + rm /tmp/tmp.JcCQ9RsBN5 /tmp/tmp.QqpNenGEc7 + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zUpFUsxoFn +++ mktemp ++ local LAST_ERR=/tmp/tmp.yX2PCbAkbH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.zUpFUsxoFn ++ cat /tmp/tmp.yX2PCbAkbH ++ rm /tmp/tmp.zUpFUsxoFn /tmp/tmp.yX2PCbAkbH ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Cc6uJCL1U6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.bYA3Ko2Hu6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Cc6uJCL1U6 ++ cat /tmp/tmp.bYA3Ko2Hu6 ++ rm /tmp/tmp.Cc6uJCL1U6 /tmp/tmp.bYA3Ko2Hu6 ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.e7Yn1lrWM8 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.JTgECCBJ5L +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ cat /tmp/tmp.e7Yn1lrWM8 +++++ cat /tmp/tmp.JTgECCBJ5L +++++ rm /tmp/tmp.e7Yn1lrWM8 /tmp/tmp.JTgECCBJ5L +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Tj95zqIswq +++ mktemp ++ local LAST_ERR=/tmp/tmp.sm5rsx3De3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Tj95zqIswq ++ cat /tmp/tmp.sm5rsx3De3 ++ rm /tmp/tmp.Tj95zqIswq /tmp/tmp.sm5rsx3De3 ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.t4XHPhskIW +++ mktemp ++ local LAST_ERR=/tmp/tmp.GKVnPrjF4t ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.t4XHPhskIW ++ cat /tmp/tmp.GKVnPrjF4t ++ rm /tmp/tmp.t4XHPhskIW /tmp/tmp.GKVnPrjF4t ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-fjdhh + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GJ34u6epXr +++ mktemp ++ local LAST_ERR=/tmp/tmp.tZDbZ8TNah ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.GJ34u6epXr ++ cat /tmp/tmp.tZDbZ8TNah ++ rm /tmp/tmp.GJ34u6epXr /tmp/tmp.tZDbZ8TNah ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0Hd4P8RP8i +++ mktemp ++ local LAST_ERR=/tmp/tmp.i0mxMaDw74 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.0Hd4P8RP8i ++ cat /tmp/tmp.i0mxMaDw74 ++ rm /tmp/tmp.0Hd4P8RP8i /tmp/tmp.i0mxMaDw74 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + kubectl_bin delete -f /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.RAOISeD4sY ++ mktemp + local LAST_ERR=/tmp/tmp.R7lHB4BDmw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete -f /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.RAOISeD4sY perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.R7lHB4BDmw + rm /tmp/tmp.RAOISeD4sY /tmp/tmp.R7lHB4BDmw + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.0SlkVVRgeF ++ mktemp + local LAST_ERR=/tmp/tmp.fc9QZEavXs + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pvc --all + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.0SlkVVRgeF persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.fc9QZEavXs + rm /tmp/tmp.0SlkVVRgeF /tmp/tmp.fc9QZEavXs + return 0 + desc 'PXC cluster update with the latest image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with the latest image by version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.pPI6yGJYoi ++ mktemp + local LAST_ERR=/tmp/tmp.8PseGIJ7Rp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.pPI6yGJYoi secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.8PseGIJ7Rp + rm /tmp/tmp.pPI6yGJYoi /tmp/tmp.8PseGIJ7Rp + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + kubectl_bin apply -f - ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + local LAST_OUT=/tmp/tmp.6CtE9MrZXN + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update-20861~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + local LAST_ERR=/tmp/tmp.nEsZwe1zo2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.6CtE9MrZXN deployment.apps/pxc-client unchanged + cat /tmp/tmp.nEsZwe1zo2 + rm /tmp/tmp.6CtE9MrZXN /tmp/tmp.nEsZwe1zo2 + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.1P253IQh7h + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update-20861~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + cat /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + local LAST_ERR=/tmp/tmp.xx06e6M95f + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.1P253IQh7h perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.xx06e6M95f + rm /tmp/tmp.1P253IQh7h /tmp/tmp.xx06e6M95f + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.o0EddD3UEF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TLpkvqSvSc +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.o0EddD3UEF +++ cat /tmp/tmp.TLpkvqSvSc +++ rm /tmp/tmp.o0EddD3UEF /tmp/tmp.TLpkvqSvSc +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace smart-update-haproxy-0........................................error: a container name must be specified for pod smart-update-haproxy-0, choose one of: [haproxy pxc-monit] .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-1...........................Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-2 + local container=pxc + set +o xtrace smart-update-pxc-2......................................Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nskEOIxFW6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.iqpJWw6h9F ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.nskEOIxFW6 ++ cat /tmp/tmp.iqpJWw6h9F ++ rm /tmp/tmp.nskEOIxFW6 /tmp/tmp.iqpJWw6h9F ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-5d749ff8b6-fjdhh + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cmPTCBior0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.QqR4TUeFc8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.cmPTCBior0 ++ cat /tmp/tmp.QqR4TUeFc8 ++ rm /tmp/tmp.cmPTCBior0 /tmp/tmp.QqR4TUeFc8 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Zkjea8yROv +++ mktemp ++ local LAST_ERR=/tmp/tmp.lxQeh1bB85 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Zkjea8yROv ++ cat /tmp/tmp.lxQeh1bB85 ++ rm /tmp/tmp.Zkjea8yROv /tmp/tmp.lxQeh1bB85 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-5d749ff8b6-fjdhh + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JFJCEnCGiX +++ mktemp ++ local LAST_ERR=/tmp/tmp.hsS6vrcHjS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.JFJCEnCGiX ++ cat /tmp/tmp.hsS6vrcHjS ++ rm /tmp/tmp.JFJCEnCGiX /tmp/tmp.hsS6vrcHjS ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JljfU7TUMa +++ mktemp ++ local LAST_ERR=/tmp/tmp.FG8hwXLDp5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.JljfU7TUMa ++ cat /tmp/tmp.FG8hwXLDp5 ++ rm /tmp/tmp.JljfU7TUMa /tmp/tmp.FG8hwXLDp5 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0kWmpMJvDW +++ mktemp ++ local LAST_ERR=/tmp/tmp.pnP7nw0z8D ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.0kWmpMJvDW ++ cat /tmp/tmp.pnP7nw0z8D Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.0kWmpMJvDW /tmp/tmp.pnP7nw0z8D ++ return 0 + '[' '' ']' + vs_image=latest ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jgmVbCKBqI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ezgjru8e4a +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.jgmVbCKBqI +++ cat /tmp/tmp.ezgjru8e4a +++ rm /tmp/tmp.jgmVbCKBqI /tmp/tmp.ezgjru8e4a +++ return 0 ++ client_pod=pxc-client-5d749ff8b6-fjdhh ++ wait_pod pxc-client-5d749ff8b6-fjdhh ++ local pod=pxc-client-5d749ff8b6-fjdhh ++ local max_retry=480 ++ local ns= +++ echo pxc-client-5d749ff8b6-fjdhh +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.s3xEww3OIm ++ mktemp + local LAST_ERR=/tmp/tmp.neuaGtmZhv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.s3xEww3OIm perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.neuaGtmZhv + rm /tmp/tmp.s3xEww3OIm /tmp/tmp.neuaGtmZhv + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UpjY3Vy5Yf +++ mktemp ++ local LAST_ERR=/tmp/tmp.dFup7J4MUp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.UpjY3Vy5Yf ++ cat /tmp/tmp.dFup7J4MUp ++ rm /tmp/tmp.UpjY3Vy5Yf /tmp/tmp.dFup7J4MUp ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rpWvgobY2H +++ mktemp ++ local LAST_ERR=/tmp/tmp.ltzGpEKf1z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.rpWvgobY2H ++ cat /tmp/tmp.ltzGpEKf1z ++ rm /tmp/tmp.rpWvgobY2H /tmp/tmp.ltzGpEKf1z ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.jLtD8UTie9 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.DDhTubwRMl +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ cat /tmp/tmp.jLtD8UTie9 +++++ cat /tmp/tmp.DDhTubwRMl +++++ rm /tmp/tmp.jLtD8UTie9 /tmp/tmp.DDhTubwRMl +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.58n22tgpCu +++ mktemp ++ local LAST_ERR=/tmp/tmp.OsSLP0m1y8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.58n22tgpCu ++ cat /tmp/tmp.OsSLP0m1y8 ++ rm /tmp/tmp.58n22tgpCu /tmp/tmp.OsSLP0m1y8 ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Be96FODMWl +++ mktemp ++ local LAST_ERR=/tmp/tmp.EjIFezbSMG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Be96FODMWl ++ cat /tmp/tmp.EjIFezbSMG ++ rm /tmp/tmp.Be96FODMWl /tmp/tmp.EjIFezbSMG ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dEF547PttY +++ mktemp ++ local LAST_ERR=/tmp/tmp.guNhGoX8WF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.dEF547PttY ++ cat /tmp/tmp.guNhGoX8WF ++ rm /tmp/tmp.dEF547PttY /tmp/tmp.guNhGoX8WF ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-fjdhh ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ae2LkmLpep +++ mktemp ++ local LAST_ERR=/tmp/tmp.SNiBN5VHF0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.ae2LkmLpep ++ cat /tmp/tmp.SNiBN5VHF0 ++ rm /tmp/tmp.ae2LkmLpep /tmp/tmp.SNiBN5VHF0 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + kubectl_bin delete -f /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.klue4pqqNI ++ mktemp + local LAST_ERR=/tmp/tmp.nUEmAXgAxx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete -f /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.klue4pqqNI perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.nUEmAXgAxx + rm /tmp/tmp.klue4pqqNI /tmp/tmp.nUEmAXgAxx + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.MSWF4shKrr ++ mktemp + local LAST_ERR=/tmp/tmp.8WlNsgyua9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pvc --all + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.MSWF4shKrr persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.8WlNsgyua9 + rm /tmp/tmp.MSWF4shKrr /tmp/tmp.8WlNsgyua9 + return 0 + desc 'PXC cluster update with explicitly specified image inside version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with explicitly specified image inside version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.MwBJee5V1Z ++ mktemp + local LAST_ERR=/tmp/tmp.NoW4QkArZp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.MwBJee5V1Z secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.NoW4QkArZp + rm /tmp/tmp.MwBJee5V1Z /tmp/tmp.NoW4QkArZp + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + kubectl_bin apply -f - ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + local LAST_OUT=/tmp/tmp.MnMpuPdPp8 ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update-20861~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_ERR=/tmp/tmp.v3uV4ieFgz + local exit_status=0 + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.MnMpuPdPp8 deployment.apps/pxc-client unchanged + cat /tmp/tmp.v3uV4ieFgz + rm /tmp/tmp.MnMpuPdPp8 /tmp/tmp.v3uV4ieFgz + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + '[' -z '' ']' + cat_config /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + kubectl_bin apply -f - + cat /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update-20861~ + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + local LAST_OUT=/tmp/tmp.ffUu1muV0g ++ mktemp + local LAST_ERR=/tmp/tmp.vu1PzVib2F + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.ffUu1muV0g perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.vu1PzVib2F + rm /tmp/tmp.ffUu1muV0g /tmp/tmp.vu1PzVib2F + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4kEdWMi0hM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.J8ohm0wr1P +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.4kEdWMi0hM +++ cat /tmp/tmp.J8ohm0wr1P +++ rm /tmp/tmp.4kEdWMi0hM /tmp/tmp.J8ohm0wr1P +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace smart-update-haproxy-0......................................error: a container name must be specified for pod smart-update-haproxy-0, choose one of: [haproxy pxc-monit] .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-1 ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-1.....................................Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace smart-update-pxc-2................................Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ugWb5aELlA +++ mktemp ++ local LAST_ERR=/tmp/tmp.UfhEvDyJbg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.ugWb5aELlA ++ cat /tmp/tmp.UfhEvDyJbg ++ rm /tmp/tmp.ugWb5aELlA /tmp/tmp.UfhEvDyJbg ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FhNomh7PWK +++ mktemp ++ local LAST_ERR=/tmp/tmp.tzK4KKAbJS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.FhNomh7PWK ++ cat /tmp/tmp.tzK4KKAbJS ++ rm /tmp/tmp.FhNomh7PWK /tmp/tmp.tzK4KKAbJS ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.POLxGyN1b9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.6sdDtQDDzT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.POLxGyN1b9 ++ cat /tmp/tmp.6sdDtQDDzT ++ rm /tmp/tmp.POLxGyN1b9 /tmp/tmp.6sdDtQDDzT ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xKaCuBWt3x +++ mktemp ++ local LAST_ERR=/tmp/tmp.3xJ3SnPWTr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.xKaCuBWt3x ++ cat /tmp/tmp.3xJ3SnPWTr ++ rm /tmp/tmp.xKaCuBWt3x /tmp/tmp.3xJ3SnPWTr ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TPcOIZBFtu +++ mktemp ++ local LAST_ERR=/tmp/tmp.qElhntcNex ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.TPcOIZBFtu ++ cat /tmp/tmp.qElhntcNex ++ rm /tmp/tmp.TPcOIZBFtu /tmp/tmp.qElhntcNex ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aggtdbQsg5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.AccprGEAwg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.aggtdbQsg5 ++ cat /tmp/tmp.AccprGEAwg Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.aggtdbQsg5 /tmp/tmp.AccprGEAwg ++ return 0 + '[' '' ']' +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ grep :8.0 ++ sort -V ++ head -n1 ++++ grep -c percona-xtradb-cluster-operator ++ tail -n2 ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fR4EdSReHE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Mz5ytBf6ya +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.fR4EdSReHE +++ cat /tmp/tmp.Mz5ytBf6ya +++ rm /tmp/tmp.fR4EdSReHE /tmp/tmp.Mz5ytBf6ya +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-5699d7755d-lkztd -n pxc-operator -- curl -s http://version-service.smart-update-20861.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.ymw3VElLCA +++ mktemp ++ local LAST_ERR=/tmp/tmp.HHCHAiR0Yf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl exec -ti percona-xtradb-cluster-operator-5699d7755d-lkztd -n pxc-operator -- curl -s http://version-service.smart-update-20861.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.ymw3VElLCA ++ cat /tmp/tmp.HHCHAiR0Yf Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.ymw3VElLCA /tmp/tmp.HHCHAiR0Yf ++ return 0 + vs_image=percona/percona-xtradb-cluster:8.0.19-10.1 ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MliInswws6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.a3sk2gtZZh +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.MliInswws6 +++ cat /tmp/tmp.a3sk2gtZZh +++ rm /tmp/tmp.MliInswws6 /tmp/tmp.a3sk2gtZZh +++ return 0 ++ client_pod=pxc-client-5d749ff8b6-fjdhh ++ wait_pod pxc-client-5d749ff8b6-fjdhh ++ local pod=pxc-client-5d749ff8b6-fjdhh ++ local max_retry=480 ++ local ns= +++ echo pxc-client-5d749ff8b6-fjdhh +++ egrep '^(pxc|proxysql)$' +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ local container= ++ set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.19-10.1","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.0fYtJGe5iZ ++ mktemp + local LAST_ERR=/tmp/tmp.WbqBoMvhYM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.19-10.1","schedule": "* * * * *"}}}' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.0fYtJGe5iZ perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.WbqBoMvhYM + rm /tmp/tmp.0fYtJGe5iZ /tmp/tmp.WbqBoMvhYM + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.19-10.1 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.19-10.1 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pR0e6GWGd4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.luGdhQozff ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.pR0e6GWGd4 ++ cat /tmp/tmp.luGdhQozff ++ rm /tmp/tmp.pR0e6GWGd4 /tmp/tmp.luGdhQozff ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LiOdyMKc0c +++ mktemp ++ local LAST_ERR=/tmp/tmp.YTNyssHyA4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.LiOdyMKc0c ++ cat /tmp/tmp.YTNyssHyA4 ++ rm /tmp/tmp.LiOdyMKc0c /tmp/tmp.YTNyssHyA4 ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.pGcL7Woo1M ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.S66J6aPPgZ +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ cat /tmp/tmp.pGcL7Woo1M +++++ cat /tmp/tmp.S66J6aPPgZ +++++ rm /tmp/tmp.pGcL7Woo1M /tmp/tmp.S66J6aPPgZ +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EZbUcIoytW +++ mktemp ++ local LAST_ERR=/tmp/tmp.JNokM1Ay7c ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.EZbUcIoytW ++ cat /tmp/tmp.JNokM1Ay7c ++ rm /tmp/tmp.EZbUcIoytW /tmp/tmp.JNokM1Ay7c ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BqZ46EHsNo +++ mktemp ++ local LAST_ERR=/tmp/tmp.BkT1ZUM3BW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.BqZ46EHsNo ++ cat /tmp/tmp.BkT1ZUM3BW ++ rm /tmp/tmp.BqZ46EHsNo /tmp/tmp.BkT1ZUM3BW ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GVHXj69CNA +++ mktemp ++ local LAST_ERR=/tmp/tmp.1dJ5jvLjSW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.GVHXj69CNA ++ cat /tmp/tmp.1dJ5jvLjSW ++ rm /tmp/tmp.GVHXj69CNA /tmp/tmp.1dJ5jvLjSW ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Aq2V9AXywI +++ mktemp ++ local LAST_ERR=/tmp/tmp.Qiou2F00k8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Aq2V9AXywI ++ cat /tmp/tmp.Qiou2F00k8 ++ rm /tmp/tmp.Aq2V9AXywI /tmp/tmp.Qiou2F00k8 ++ return 0 + client_pod=pxc-client-5d749ff8b6-fjdhh + wait_pod pxc-client-5d749ff8b6-fjdhh + local pod=pxc-client-5d749ff8b6-fjdhh + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-fjdhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-fjdhh.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.FUQtM3DI37/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/compare/select-1.sql /tmp/tmp.FUQtM3DI37/select-1.sql + kubectl_bin delete -f /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.bADwz6wFi5 ++ mktemp + local LAST_ERR=/tmp/tmp.BRZghE38bn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete -f /tmp/tmp.FUQtM3DI37/smart-update-version-service-unreachable.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.bADwz6wFi5 perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.BRZghE38bn + rm /tmp/tmp.bADwz6wFi5 /tmp/tmp.BRZghE38bn + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.4lUHJCtpQG ++ mktemp + local LAST_ERR=/tmp/tmp.rmzMt97iqU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pvc --all + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.4lUHJCtpQG persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.rmzMt97iqU + rm /tmp/tmp.4lUHJCtpQG /tmp/tmp.rmzMt97iqU + return 0 + desc cleanup + set +o xtrace ----------------------------------------------------------------------------------- cleanup ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.FDYAh0OaEk ++ mktemp + local LAST_ERR=/tmp/tmp.9mlS3A9rWA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/smart-update/conf/vs.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.FDYAh0OaEk deployment.apps "version-service" deleted service "version-service" deleted + cat /tmp/tmp.9mlS3A9rWA + rm /tmp/tmp.FDYAh0OaEk /tmp/tmp.9mlS3A9rWA + return 0 + destroy smart-update-20861 + local namespace=smart-update-20861 + local ignore_logs=false + [[ false == \f\a\l\s\e ]] + grep -v 'get backup status: Job.batch' + tee /tmp/tmp.FUQtM3DI37/operator.log + grep -v 'the object has been modified' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + sort -u + grep -v level=info ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.iAVlkj7YkL +++ mktemp ++ local LAST_ERR=/tmp/tmp.zBzS7mavUi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.iAVlkj7YkL ++ cat /tmp/tmp.zBzS7mavUi ++ rm /tmp/tmp.iAVlkj7YkL /tmp/tmp.zBzS7mavUi ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-5699d7755d-lkztd ++ mktemp + local LAST_OUT=/tmp/tmp.8ThtXDFqAo ++ mktemp + local LAST_ERR=/tmp/tmp.eUvnYbIEZx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-5699d7755d-lkztd + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.8ThtXDFqAo + cat /tmp/tmp.eUvnYbIEZx + rm /tmp/tmp.8ThtXDFqAo /tmp/tmp.eUvnYbIEZx + return 0 I0412 10:53:37.482893 1 request.go:645] Throttling request took 1.043729961s, request: GET:https://10.67.240.1:443/apis/pxc.percona.com/v1-7-0?timeout=32s {"level":"error",,"caller":"pxc/version.go:100","msg":"failed to ensure version","error":"failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?customResourceUid=9499930f-7ae1-4192-9e9a-e12c7299af99&databaseVersion=8.0.19-10&kubeVersion=v1.20.15-gke.4100&platform=kubernetes\": dial tcp: lookup version-service on 10.67.240.10:53: no such host","errorVerbose":"Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?customResourceUid=9499930f-7ae1-4192-9e9a-e12c7299af99&databaseVersion=8.0.19-10&kubeVersion=v1.20.15-gke.4100&platform=kubernetes\": dial tcp: lookup version-service on 10.67.240.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:158\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).sheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:98\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/robfig/cron/v3/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/robfig/cron/v3/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581","stacktrace":"github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).sheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:100\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/robfig/cron/v3/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/robfig/cron/v3/cron.go:312"} {"level":"error",,"caller":"pxc/version.go:100","msg":"failed to ensure version","error":"failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.19-10.1?customResourceUid=60fe7f9e-cf19-4832-a9dc-c7f71ba44d74&databaseVersion=8.0.19-10&kubeVersion=v1.20.15-gke.4100&platform=kubernetes\": dial tcp: lookup version-service on 10.67.240.10:53: no such host","errorVerbose":"Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.19-10.1?customResourceUid=60fe7f9e-cf19-4832-a9dc-c7f71ba44d74&databaseVersion=8.0.19-10&kubeVersion=v1.20.15-gke.4100&platform=kubernetes\": dial tcp: lookup version-service on 10.67.240.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:158\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).sheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:98\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/robfig/cron/v3/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/robfig/cron/v3/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581","stacktrace":"github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).sheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:100\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/robfig/cron/v3/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/robfig/cron/v3/cron.go:312"} {"level":"error",,"caller":"pxc/version.go:100","msg":"failed to ensure version","error":"failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?customResourceUid=71d8ea66-71ad-45be-921b-3a1ceb24299d&databaseVersion=8.0.19-10&kubeVersion=v1.20.15-gke.4100&platform=kubernetes\": dial tcp: lookup version-service on 10.67.240.10:53: no such host","errorVerbose":"Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?customResourceUid=71d8ea66-71ad-45be-921b-3a1ceb24299d&databaseVersion=8.0.19-10&kubeVersion=v1.20.15-gke.4100&platform=kubernetes\": dial tcp: lookup version-service on 10.67.240.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:158\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).sheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:98\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/robfig/cron/v3/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/robfig/cron/v3/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581","stacktrace":"github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).sheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:100\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/robfig/cron/v3/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/vendor/github.com/robfig/cron/v3/cron.go:312"} {"level":"info",,"caller":"pxc/controller.go:468","msg":"reconcile replication error","err":"get primary pxc pod: not found"} {"level":"info",,"caller":"pxc/upgrade.go:266","msg":"statefulSet was changed, run smart update"} {"level":"info",,"caller":"pxc/upgrade.go:294","msg":"primary pod","pod name":"smart-update-pxc-0.smart-update-pxc.smart-update-20861"} {"level":"info",,"caller":"pxc/upgrade.go:294","msg":"primary pod","pod name":"smart-update-pxc-0.smart-update-pxc.smart-update-20861.svc.cluster.local"} {"level":"info",,"caller":"pxc/upgrade.go:311","msg":"apply changes to secondary pod","pod name":"smart-update-pxc-1"} {"level":"info",,"caller":"pxc/upgrade.go:311","msg":"apply changes to secondary pod","pod name":"smart-update-pxc-2"} {"level":"info",,"caller":"pxc/upgrade.go:318","msg":"apply changes to primary pod","pod name":"smart-update-pxc-0"} {"level":"info",,"caller":"pxc/upgrade.go:323","msg":"smart update finished"} {"level":"info",,"caller":"pxc/upgrade.go:395","msg":"pod present in hostgroups","pod name":"smart-update-pxc-0"} {"level":"info",,"caller":"pxc/upgrade.go:395","msg":"pod present in hostgroups","pod name":"smart-update-pxc-1"} {"level":"info",,"caller":"pxc/upgrade.go:395","msg":"pod present in hostgroups","pod name":"smart-update-pxc-2"} {"level":"info",,"caller":"pxc/upgrade.go:427","msg":"pod is online","pod name":"smart-update-pxc-0"} {"level":"info",,"caller":"pxc/upgrade.go:427","msg":"pod is online","pod name":"smart-update-pxc-1"} {"level":"info",,"caller":"pxc/upgrade.go:427","msg":"pod is online","pod name":"smart-update-pxc-2"} {"level":"info",,"caller":"pxc/upgrade.go:588","msg":"pod is running","pod name":"smart-update-pxc-0"} {"level":"info",,"caller":"pxc/upgrade.go:588","msg":"pod is running","pod name":"smart-update-pxc-1"} {"level":"info",,"caller":"pxc/upgrade.go:588","msg":"pod is running","pod name":"smart-update-pxc-2"} {"level":"info",,"caller":"pxc/version.go:107","msg":"add new job","name":"ensure-version/smart-update-20861/smart-update","schedule":"* * * * *"} {"level":"info",,"caller":"pxc/version.go:328","msg":"update PXC version (fetched from db)","new version":"8.0.19-10"} {"level":"info",,"caller":"pxc/version.go:328","msg":"update PXC version (fetched from db)","new version":"8.0.27-18.1"} {"level":"info",,"caller":"pxc/version.go:65","msg":"add new job","schedule":"* * * * *"} {"level":"info",,"caller":"pxc/version.go:77","msg":"cluster is not found, deleting the job","job nameError":"json: unsupported type: func(*v1.PerconaXtraDBCluster) string","cluster":"smart-update","namespace":"smart-update-20861"} {"level":"info",,"caller":"v1/pxc_types.go:874","msg":"HAProxy size will be changed from 1 to 2 due to safe config"} {"level":"info",,"caller":"v1/pxc_types.go:874","msg":"ProxySQL size will be changed from 1 to 2 due to safe config"} {"level":"info",,"caller":"v1/pxc_types.go:875","msg":"Set allowUnsafeConfigurations=true to disable safe configuration"} {"level":"info",,"logger":"cmd","msg":"Git commit: 706f792ae47c369cb3556faff186b6873a8a247f Git branch: PR-1125-706f792a Build time: 2022-04-12T09:09:41Z"} {"level":"info",,"logger":"cmd","msg":"Go OS/Arch: linux/amd64"} {"level":"info",,"logger":"cmd","msg":"Go Version: go1.17.8"} {"level":"info",,"logger":"cmd","msg":"operator-sdk Version: v0.19.4"} {"level":"info",,"logger":"cmd","msg":"Registering Components."} {"level":"info",,"logger":"cmd","msg":"Runs on","platform":"kubernetes","version":"v1.20.15-gke.4100"} {"level":"info",,"logger":"cmd","msg":"Starting the Cmd."} {"level":"info",,"logger":"controller-runtime.certwatcher","msg":"Starting certificate watcher"} {"level":"info",,"logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate"} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbclusterbackup-controller","msg":"Starting Controller"} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbclusterbackup-controller","msg":"Starting EventSource","source":"kind source: /, Kind="} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbclusterbackup-controller","msg":"Starting workers","worker count":1} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbcluster-controller","msg":"Starting Controller"} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbcluster-controller","msg":"Starting EventSource","source":"kind source: /, Kind="} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbcluster-controller","msg":"Starting workers","worker count":1} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbclusterrestore-controller","msg":"Starting Controller"} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbclusterrestore-controller","msg":"Starting EventSource","source":"kind source: /, Kind="} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbclusterrestore-controller","msg":"Starting workers","worker count":1} {"level":"info",,"logger":"controller-runtime.manager","msg":"starting metrics server","path":"/metrics"} {"level":"info",,"logger":"controller-runtime.metrics","msg":"metrics server is starting to listen","addr":":8080"} {"level":"info",,"logger":"controller-runtime.webhook","msg":"registering webhook","path":"/validate-percona-xtradbcluster"} {"level":"info",,"logger":"controller-runtime.webhook","msg":"serving webhook server","host":"","port":9443} {"level":"info",,"logger":"controller-runtime.webhook.webhooks","msg":"starting webhook server"} {"level":"info",,"logger":"leader","msg":"Became the leader."} {"level":"info",,"logger":"leader","msg":"No pre-existing lock was found."} {"level":"info",,"logger":"leader","msg":"Trying to become the leader."} + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name, label selector, or --all flag specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.0AduSU3fae ++ mktemp + local LAST_ERR=/tmp/tmp.sWeDxVGFBD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.0AduSU3fae No resources found + cat /tmp/tmp.sWeDxVGFBD + rm /tmp/tmp.0AduSU3fae /tmp/tmp.sWeDxVGFBD + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.YIAWpFACO4 ++ mktemp + local LAST_ERR=/tmp/tmp.Qqi6aN0GsT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.YIAWpFACO4 No resources found + cat /tmp/tmp.Qqi6aN0GsT + rm /tmp/tmp.YIAWpFACO4 /tmp/tmp.Qqi6aN0GsT + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.Oh3lparWTn ++ mktemp + local LAST_ERR=/tmp/tmp.GRDOLqD50U + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.Oh3lparWTn No resources found + cat /tmp/tmp.GRDOLqD50U + rm /tmp/tmp.Oh3lparWTn /tmp/tmp.GRDOLqD50U + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.Aw9vtV2DvN ++ mktemp + local LAST_ERR=/tmp/tmp.fazb9Q7lr3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.Aw9vtV2DvN validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.fazb9Q7lr3 + rm /tmp/tmp.Aw9vtV2DvN /tmp/tmp.fazb9Q7lr3 + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted namespace "cert-manager" deleted serviceaccount "cert-manager-cainjector" deleted serviceaccount "cert-manager" deleted serviceaccount "cert-manager-webhook" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted role.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted rolebinding.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted service "cert-manager" deleted service "cert-manager-webhook" deleted deployment.apps "cert-manager-cainjector" deleted deployment.apps "cert-manager" deleted deployment.apps "cert-manager-webhook" deleted mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + rm -rf /tmp/tmp.FUQtM3DI37 + kubectl_bin delete --grace-period=0 --force=true namespace smart-update-20861 + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.itTd7b6r7R + local LAST_OUT=/tmp/tmp.QCwSIO6jgF ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.S8Dv1ELhLx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete --grace-period=0 --force=true namespace smart-update-20861 + local LAST_ERR=/tmp/tmp.G01VRa0CTv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete --grace-period=0 --force=true namespace pxc-operator