Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/logs/smart-update2-8-0.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + API=pxc.percona.com/v9-9-9 + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + CLUSTER=smart-update + CLUSTER_SIZE=3 + PROXY_SIZE=2 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0 ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' + PXC_VER=8.0 + TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + VS_URL=http://version-service + VS_PORT=11000 + VS_ENDPOINT=http://version-service:11000 + [[ 8.0 == \8\.\4 ]] + VS_UPDATE_STRATEGY=recommended + main + create_infra smart-update2-30577 + local ns=smart-update2-30577 + '[' -n pxc-operator ']' + read -r pns name + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrole ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.0M8iIWtlEI + kubectl_bin get ns ++ mktemp + local LAST_ERR=/tmp/tmp.9CDflTN5qF + local exit_status=0 ++ mktemp + local LAST_OUT=/tmp/tmp.BNzTAbvL7x ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator ++ mktemp + local LAST_ERR=/tmp/tmp.l0XVbVLaH5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BNzTAbvL7x + cat /tmp/tmp.l0XVbVLaH5 + rm /tmp/tmp.BNzTAbvL7x /tmp/tmp.l0XVbVLaH5 + return 0 namespace "cert-manager" deleted namespace "smart-update2-2003" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0M8iIWtlEI namespace "pxc-operator" deleted + cat /tmp/tmp.9CDflTN5qF + rm /tmp/tmp.0M8iIWtlEI /tmp/tmp.9CDflTN5qF + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.vgbeq1I2V2 ++ mktemp + local LAST_ERR=/tmp/tmp.hziJqUnF41 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vgbeq1I2V2 namespace/pxc-operator created + cat /tmp/tmp.hziJqUnF41 + rm /tmp/tmp.vgbeq1I2V2 /tmp/tmp.hziJqUnF41 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZZkELZ4u5t +++ mktemp ++ local LAST_ERR=/tmp/tmp.HbnuDRC35i ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZZkELZ4u5t ++ cat /tmp/tmp.HbnuDRC35i ++ rm /tmp/tmp.ZZkELZ4u5t /tmp/tmp.HbnuDRC35i ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster6 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.EIYdU7sPil ++ mktemp + local LAST_ERR=/tmp/tmp.GLaJ3pKL4g + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster6 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EIYdU7sPil Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster6" modified. + cat /tmp/tmp.GLaJ3pKL4g + rm /tmp/tmp.EIYdU7sPil /tmp/tmp.GLaJ3pKL4g + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.jdSUODyiFA ++ mktemp + local LAST_ERR=/tmp/tmp.gWfwaauBF0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jdSUODyiFA customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.gWfwaauBF0 + rm /tmp/tmp.jdSUODyiFA /tmp/tmp.gWfwaauBF0 + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.dcFS1qeqhL ++ mktemp + local LAST_ERR=/tmp/tmp.MceVih2drX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dcFS1qeqhL customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.MceVih2drX E0702 04:22:34.340962 22203 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusters.pxc.percona.com&resourceVersion=1782966151100383007&timeoutSeconds=312&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.dcFS1qeqhL /tmp/tmp.MceVih2drX + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.O3xsUATaLy ++ mktemp + local LAST_ERR=/tmp/tmp.OXhUJnvDoA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.O3xsUATaLy customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.OXhUJnvDoA E0702 04:22:37.325933 22596 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterbackups.pxc.percona.com&resourceVersion=1782966156105326000&timeoutSeconds=548&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.O3xsUATaLy /tmp/tmp.OXhUJnvDoA + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.uYXVg7C3f7 ++ mktemp + local LAST_ERR=/tmp/tmp.1aCy1jVePX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uYXVg7C3f7 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.1aCy1jVePX E0702 04:22:39.862284 23101 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterrestores.pxc.percona.com&resourceVersion=1782966156105326000&timeoutSeconds=436&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.uYXVg7C3f7 /tmp/tmp.1aCy1jVePX + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/cw-rbac.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.7VcLGE9Zww ++ mktemp + local LAST_ERR=/tmp/tmp.oqMMOxrFw4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7VcLGE9Zww clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.oqMMOxrFw4 + rm /tmp/tmp.7VcLGE9Zww /tmp/tmp.oqMMOxrFw4 + return 0 + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/cw-operator.yaml + local LAST_OUT=/tmp/tmp.Qbpbu9J9s6 ++ mktemp + local LAST_ERR=/tmp/tmp.v1Hdjx3Mbk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Qbpbu9J9s6 deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.v1Hdjx3Mbk + rm /tmp/tmp.Qbpbu9J9s6 /tmp/tmp.v1Hdjx3Mbk + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.dMNLn2KJU2 ++ mktemp + local LAST_ERR=/tmp/tmp.T3GnakLFEj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dMNLn2KJU2 pod/percona-xtradb-cluster-operator-59848dbdcc-fqrfj condition met + cat /tmp/tmp.T3GnakLFEj E0702 04:23:01.619691 26325 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-59848dbdcc-fqrfj&resourceVersion=1782966181261820000&timeoutSeconds=439&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.dMNLn2KJU2 /tmp/tmp.T3GnakLFEj + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ head -1 ++ local LAST_OUT=/tmp/tmp.FKYr2iALGC +++ mktemp ++ local LAST_ERR=/tmp/tmp.C1HyEr0Hg2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FKYr2iALGC ++ cat /tmp/tmp.C1HyEr0Hg2 ++ rm /tmp/tmp.FKYr2iALGC /tmp/tmp.C1HyEr0Hg2 ++ return 0 + wait_pod percona-xtradb-cluster-operator-59848dbdcc-fqrfj 480 pxc-operator + local pod=percona-xtradb-cluster-operator-59848dbdcc-fqrfj + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-59848dbdcc-fqrfj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-59848dbdcc-fqrfj condition met E0702 04:23:04.951952 26842 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-59848dbdcc-fqrfj&resourceVersion=1782966184193136000&timeoutSeconds=349&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-59848dbdcc-fqrfj to become Ready.Ok + sleep 3 + create_namespace smart-update2-30577 + local namespace=smart-update2-30577 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl api-resources ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrole ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' ++ mktemp + desc 'cleaned up old namespaces smart-update2-30577' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces smart-update2-30577 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace smart-update2-30577 + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.iRqu5Q9Kl7 ++ mktemp + local LAST_ERR=/tmp/tmp.eUrtanPqtR + local exit_status=0 ++ seq 0 2 + local LAST_OUT=/tmp/tmp.XI2eN3pxYr + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-30577 + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + local LAST_ERR=/tmp/tmp.oG4Y8j64xj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XI2eN3pxYr + cat /tmp/tmp.oG4Y8j64xj + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + rm /tmp/tmp.XI2eN3pxYr /tmp/tmp.oG4Y8j64xj + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-30577 + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-30577 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.iRqu5Q9Kl7 + cat /tmp/tmp.eUrtanPqtR Error from server (NotFound): namespaces "smart-update2-30577" not found + rm /tmp/tmp.iRqu5Q9Kl7 /tmp/tmp.eUrtanPqtR + return 1 + : + wait_for_delete namespace/smart-update2-30577 + local res=namespace/smart-update2-30577 + echo -n 'waiting for namespace/smart-update2-30577 to be deleted' waiting for namespace/smart-update2-30577 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "smart-update2-30577" not found + desc 'create namespace smart-update2-30577' + set +o xtrace ----------------------------------------------------------------------------------- create namespace smart-update2-30577 ----------------------------------------------------------------------------------- + kubectl_bin create namespace smart-update2-30577 ++ mktemp + local LAST_OUT=/tmp/tmp.8VXWHyLqC0 ++ mktemp + local LAST_ERR=/tmp/tmp.rIfhD0aFd1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace smart-update2-30577 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8VXWHyLqC0 namespace/smart-update2-30577 created + cat /tmp/tmp.rIfhD0aFd1 + rm /tmp/tmp.8VXWHyLqC0 /tmp/tmp.rIfhD0aFd1 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.erN6hKjPMC +++ mktemp ++ local LAST_ERR=/tmp/tmp.IJbztg5OvH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.erN6hKjPMC ++ cat /tmp/tmp.IJbztg5OvH ++ rm /tmp/tmp.erN6hKjPMC /tmp/tmp.IJbztg5OvH ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster6 --namespace=smart-update2-30577 ++ mktemp + local LAST_OUT=/tmp/tmp.tEIyl8WwY0 ++ mktemp + local LAST_ERR=/tmp/tmp.qWj2FE1SHo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster6 --namespace=smart-update2-30577 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tEIyl8WwY0 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster6" modified. + cat /tmp/tmp.qWj2FE1SHo + rm /tmp/tmp.tEIyl8WwY0 /tmp/tmp.qWj2FE1SHo + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.o7aTVlOt7h ++ mktemp + local LAST_ERR=/tmp/tmp.IgiYjTCXJJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.o7aTVlOt7h secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.IgiYjTCXJJ + rm /tmp/tmp.o7aTVlOt7h /tmp/tmp.IgiYjTCXJJ + return 0 + deploy_version_service + desc 'install version service' + set +o xtrace ----------------------------------------------------------------------------------- install version service ----------------------------------------------------------------------------------- + kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json ++ mktemp + local LAST_OUT=/tmp/tmp.xvFGDI9iff ++ mktemp + local LAST_ERR=/tmp/tmp.x4ZdsJFuiD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xvFGDI9iff configmap/versions created + cat /tmp/tmp.x4ZdsJFuiD + rm /tmp/tmp.xvFGDI9iff /tmp/tmp.x4ZdsJFuiD + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.PdTDMZvA3O ++ mktemp + local LAST_ERR=/tmp/tmp.xvmBXURvVM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PdTDMZvA3O deployment.apps/version-service created service/version-service created + cat /tmp/tmp.xvmBXURvVM + rm /tmp/tmp.PdTDMZvA3O /tmp/tmp.xvmBXURvVM + return 0 + sleep 10 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.uexusOYBaR ++ mktemp + local LAST_ERR=/tmp/tmp.cihkuS2NYY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uexusOYBaR namespace/cert-manager created + cat /tmp/tmp.cihkuS2NYY + rm /tmp/tmp.uexusOYBaR /tmp/tmp.cihkuS2NYY + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.9YTECTKptw ++ mktemp + local LAST_ERR=/tmp/tmp.hh0kqTMLzQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9YTECTKptw namespace/cert-manager labeled + cat /tmp/tmp.hh0kqTMLzQ + rm /tmp/tmp.9YTECTKptw /tmp/tmp.hh0kqTMLzQ + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.bMQdAAcfhF ++ mktemp + local LAST_ERR=/tmp/tmp.JUDFhOSPae + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bMQdAAcfhF namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.JUDFhOSPae Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.bMQdAAcfhF /tmp/tmp.JUDFhOSPae + return 0 + '[' '' == 4.10 ']' + kubectl_bin -n cert-manager rollout status deployment/cert-manager --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.DnJY49YMs3 ++ mktemp + local LAST_ERR=/tmp/tmp.g7HEdkWEz5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n cert-manager rollout status deployment/cert-manager --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DnJY49YMs3 deployment "cert-manager" successfully rolled out + cat /tmp/tmp.g7HEdkWEz5 + rm /tmp/tmp.DnJY49YMs3 /tmp/tmp.g7HEdkWEz5 + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.5RO85nHv9K ++ mktemp + local LAST_ERR=/tmp/tmp.wuE9ZZRQpn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5RO85nHv9K deployment "cert-manager-cainjector" successfully rolled out + cat /tmp/tmp.wuE9ZZRQpn + rm /tmp/tmp.5RO85nHv9K /tmp/tmp.wuE9ZZRQpn + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.vi6D0pNzAX ++ mktemp + local LAST_ERR=/tmp/tmp.oOGI3J6UZC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vi6D0pNzAX Waiting for deployment "cert-manager-webhook" rollout to finish: 0 of 1 updated replicas are available... deployment "cert-manager-webhook" successfully rolled out + cat /tmp/tmp.oOGI3J6UZC + rm /tmp/tmp.vi6D0pNzAX /tmp/tmp.oOGI3J6UZC + return 0 + log 'waiting for cert-manager webhook caBundle to be injected' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:24:25+0000]' waiting for cert-manager webhook caBundle to be injected [2026-07-02T04:24:25+0000] waiting for cert-manager webhook caBundle to be injected ++ date +%s + local deadline=1782966385 + grep -q . + kubectl_bin get validatingwebhookconfigurations cert-manager-webhook -o 'jsonpath={.webhooks[0].clientConfig.caBundle}' + log 'probing cert-manager webhook' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:24:26+0000]' probing cert-manager webhook [2026-07-02T04:24:26+0000] probing cert-manager webhook + local probe_file ++ mktemp + probe_file=/tmp/tmp.PMFsfMsS21 + cat + local retries=0 + kubectl_bin apply -f /tmp/tmp.PMFsfMsS21 --dry-run=server + rm -f /tmp/tmp.PMFsfMsS21 ++ tail -n3 ++ head -n1 ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ grep :8.0 ++ sort -V +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++++ grep -c percona-xtradb-cluster-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ head -1 +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MNFLP7wKDW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gOLgxKtboC +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.MNFLP7wKDW +++ cat /tmp/tmp.gOLgxKtboC +++ rm /tmp/tmp.MNFLP7wKDW /tmp/tmp.gOLgxKtboC +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-59848dbdcc-fqrfj -n pxc-operator -- curl -s http://version-service.smart-update2-30577.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.m0M1Q0RTg8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.MgwEmMb4BU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-59848dbdcc-fqrfj -n pxc-operator -- curl -s http://version-service.smart-update2-30577.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.m0M1Q0RTg8 ++ cat /tmp/tmp.MgwEmMb4BU Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.m0M1Q0RTg8 /tmp/tmp.MgwEmMb4BU ++ return 0 + IMAGE_PXC=percona/percona-xtradb-cluster:8.0.41-32.1 + kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' ++ mktemp + local LAST_OUT=/tmp/tmp.osLiYY934U ++ mktemp + local LAST_ERR=/tmp/tmp.BwMllZ8O3i + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.osLiYY934U customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched + cat /tmp/tmp.BwMllZ8O3i + rm /tmp/tmp.osLiYY934U /tmp/tmp.BwMllZ8O3i + return 0 + kubectl_bin -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-30577.svc.cluster.local:11000 ++ mktemp + local LAST_OUT=/tmp/tmp.eJlnAa48Tr ++ mktemp + local LAST_ERR=/tmp/tmp.e3LAg3dWNQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-30577.svc.cluster.local:11000 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eJlnAa48Tr deployment.apps/percona-xtradb-cluster-operator env updated + cat /tmp/tmp.e3LAg3dWNQ + rm /tmp/tmp.eJlnAa48Tr /tmp/tmp.e3LAg3dWNQ + return 0 + desc 'Starting telemetry testing' + set +o xtrace ----------------------------------------------------------------------------------- Starting telemetry testing ----------------------------------------------------------------------------------- + /usr/bin/sed s/version-service/version-service-cr/g /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/vs.yml + kubectl_bin apply -f - ++ mktemp ++ yq 'select(.kind == "Deployment").spec.template.spec.containers[0].image' /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/vs.yml + local LAST_OUT=/tmp/tmp.a7x2sAmNSF ++ mktemp + local LAST_ERR=/tmp/tmp.MWnlt1RjQG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + yq eval '(. | select(.kind == "Deployment") | .spec.template.spec.containers[0].image) = "perconalab/version-service:main-latest"' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.a7x2sAmNSF deployment.apps/version-service-cr created service/version-service-cr created + cat /tmp/tmp.MWnlt1RjQG + rm /tmp/tmp.a7x2sAmNSF /tmp/tmp.MWnlt1RjQG + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.UNBMszAFIJ ++ mktemp + local LAST_ERR=/tmp/tmp.0wH8U00Q8q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UNBMszAFIJ pod "version-service-559c4df55c-sml9c" deleted from smart-update2-30577 namespace + cat /tmp/tmp.0wH8U00Q8q + rm /tmp/tmp.UNBMszAFIJ /tmp/tmp.0wH8U00Q8q + return 0 ++ sed -r 's/^.*:([0-9]+.[0-9]+).*/\1/' ++ echo -n percona/percona-xtradb-cluster:8.0.41-32.1 + IMAGE_PREFIX=8.0 + desc 'Enable telemetry on operator level' + set +o xtrace ----------------------------------------------------------------------------------- Enable telemetry on operator level ----------------------------------------------------------------------------------- + kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "false"' + kubectl_bin apply -n pxc-operator -f - ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.3LLm9RWi7c + local LAST_OUT=/tmp/tmp.DzfO7Vuy6h ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.ED1kNYcMT8 + local exit_status=0 + local LAST_ERR=/tmp/tmp.3IdOef0q1c + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n pxc-operator -f - + for i in '$(seq 0 2)' + set +e + kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DzfO7Vuy6h + cat /tmp/tmp.3IdOef0q1c + rm /tmp/tmp.DzfO7Vuy6h /tmp/tmp.3IdOef0q1c + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3LLm9RWi7c deployment.apps/percona-xtradb-cluster-operator configured + cat /tmp/tmp.ED1kNYcMT8 + rm /tmp/tmp.3LLm9RWi7c /tmp/tmp.ED1kNYcMT8 + return 0 + sleep 30 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.u0dVGGSFDI +++ mktemp ++ local LAST_ERR=/tmp/tmp.M1rLhdx97N ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u0dVGGSFDI ++ cat /tmp/tmp.M1rLhdx97N ++ rm /tmp/tmp.u0dVGGSFDI /tmp/tmp.M1rLhdx97N ++ return 0 + wait_pod percona-xtradb-cluster-operator-7c4f448b56-bhq8j 480 pxc-operator + local pod=percona-xtradb-cluster-operator-7c4f448b56-bhq8j + local max_retry=480 + local ns=pxc-operator ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo percona-xtradb-cluster-operator-7c4f448b56-bhq8j + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-7c4f448b56-bhq8j condition met E0702 04:25:26.107937 14564 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-7c4f448b56-bhq8j&resourceVersion=1782966323894270000&timeoutSeconds=595&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-7c4f448b56-bhq8j to become Ready.Ok + check_telemetry_transfer http://version-service-cr.smart-update2-30577.svc.cluster.local:11000 disabled enabled + local cr_vs_uri=http://version-service-cr.smart-update2-30577.svc.cluster.local:11000 + local cr_vs_channel=disabled + local telemetry_state=enabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.bwiKFYXzWM ++ mktemp + local LAST_ERR=/tmp/tmp.aVnRN5E08L + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bwiKFYXzWM deployment.apps/pxc-client created + cat /tmp/tmp.aVnRN5E08L + rm /tmp/tmp.bwiKFYXzWM /tmp/tmp.aVnRN5E08L + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.q8hpYZdLAW ++ mktemp + local LAST_ERR=/tmp/tmp.qu4wUEwEFA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.q8hpYZdLAW secret/minimal-cluster created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.qu4wUEwEFA + rm /tmp/tmp.q8hpYZdLAW /tmp/tmp.qu4wUEwEFA + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/cr-minimal.yaml + yq eval '.spec.crVersion="9.9.9"' + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858"' + yq eval '.spec.upgradeOptions.apply="disabled"' + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-30577.svc.cluster.local:11000"' + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.41-32.1"' + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.dvkmdTjP3q ++ mktemp + local LAST_ERR=/tmp/tmp.8izSe6Nt6p + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dvkmdTjP3q perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.8izSe6Nt6p + rm /tmp/tmp.dvkmdTjP3q /tmp/tmp.8izSe6Nt6p + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-pxc ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-pxc ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.AQtPT4EOZ5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JOEg2CiNev +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.AQtPT4EOZ5 +++ cat /tmp/tmp.JOEg2CiNev +++ rm /tmp/tmp.AQtPT4EOZ5 /tmp/tmp.JOEg2CiNev +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-haproxy ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-haproxy ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo minimal-cluster-haproxy-0 + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met E0702 04:27:36.122914 927 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dminimal-cluster-haproxy-0&resourceVersion=1782966452930307000&timeoutSeconds=359&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iuOSemjo68 +++ mktemp ++ local LAST_ERR=/tmp/tmp.D0UCJF7UJS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iuOSemjo68 ++ cat /tmp/tmp.D0UCJF7UJS ++ rm /tmp/tmp.iuOSemjo68 /tmp/tmp.D0UCJF7UJS ++ return 0 + client_pod=pxc-client-67fc4995bb-lhbhm + wait_pod pxc-client-67fc4995bb-lhbhm + local pod=pxc-client-67fc4995bb-lhbhm + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-67fc4995bb-lhbhm ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-lhbhm condition met E0702 04:27:51.164803 3111 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-lhbhm&resourceVersion=1782966467729784000&timeoutSeconds=399&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-lhbhm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nU93eiSIDW +++ mktemp ++ local LAST_ERR=/tmp/tmp.h1yKFfWx8R ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nU93eiSIDW ++ cat /tmp/tmp.h1yKFfWx8R ++ rm /tmp/tmp.nU93eiSIDW /tmp/tmp.h1yKFfWx8R ++ return 0 + client_pod=pxc-client-67fc4995bb-lhbhm + wait_pod pxc-client-67fc4995bb-lhbhm + local pod=pxc-client-67fc4995bb-lhbhm + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-67fc4995bb-lhbhm ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-lhbhm condition met E0702 04:28:16.056044 6946 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-lhbhm&resourceVersion=1782966492135723000&timeoutSeconds=592&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-lhbhm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.start_time")' + grep -Eo '\{.*\}' + jq 'del(."grpc.time_ms")' + grep -E 'server request payload|unary call' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-cr-76d4659869-n9d8d ++ mktemp + local LAST_OUT=/tmp/tmp.Nu3d6x3mX2 ++ mktemp + local LAST_ERR=/tmp/tmp.Ssc3KmOgNf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-76d4659869-n9d8d + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Nu3d6x3mX2 + cat /tmp/tmp.Ssc3KmOgNf + rm /tmp/tmp.Nu3d6x3mX2 /tmp/tmp.Ssc3KmOgNf + return 0 + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.time_ms")' + grep -E 'server request payload|unary call' + jq 'del(."grpc.start_time")' + grep -Eo '\{.*\}' + jq 'del(."grpc.request.content".msg.customResourceUid)' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-559c4df55c-r9jqm ++ mktemp + local LAST_OUT=/tmp/tmp.P5NqmsDPxz ++ mktemp + local LAST_ERR=/tmp/tmp.BWI5gxA5GE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-559c4df55c-r9jqm + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.P5NqmsDPxz + cat /tmp/tmp.BWI5gxA5GE + rm /tmp/tmp.P5NqmsDPxz /tmp/tmp.BWI5gxA5GE + return 0 + local telemetry_log_file=enabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' disabled == disabled -a enabled == enabled ']' + desc 'operator fallback VS should have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator fallback VS should have telemetry ----------------------------------------------------------------------------------- + diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json /dev/fd/63 ++ grep -f /tmp/tmp.rCF6hRfwxh/enabled_telemetry.version-service.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json + desc 'CR VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- CR VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.rCF6hRfwxh/enabled_telemetry.version-service-cr.log.json ]] + local image_prefix=disabled + image_prefix=disabled + local telemetry_cr_log_file=enabled_telemetry.version-service-cr-disabled-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + [[ disabled == \d\i\s\a\b\l\e\d\-\r\e\c\o\m\m\e\n\d\e\d ]] + [[ disabled == \d\i\s\a\b\l\e\d\-\l\a\t\e\s\t ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' disabled == disabled -a enabled == disabled ']' + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.rh9rqrs2Ys ++ mktemp + local LAST_ERR=/tmp/tmp.d01R0zPj8s + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rh9rqrs2Ys perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.d01R0zPj8s + rm /tmp/tmp.rh9rqrs2Ys /tmp/tmp.d01R0zPj8s + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.rIASM5R7UH +++ mktemp ++ local LAST_ERR=/tmp/tmp.QHZ8rNx4nl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rIASM5R7UH ++ cat /tmp/tmp.QHZ8rNx4nl ++ rm /tmp/tmp.rIASM5R7UH /tmp/tmp.QHZ8rNx4nl ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-7c4f448b56-bhq8j ++ mktemp + local LAST_OUT=/tmp/tmp.OAhR7HjJfu ++ mktemp + local LAST_ERR=/tmp/tmp.4DhrhSEfuP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-7c4f448b56-bhq8j + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OAhR7HjJfu pod "percona-xtradb-cluster-operator-7c4f448b56-bhq8j" deleted from pxc-operator namespace + cat /tmp/tmp.4DhrhSEfuP + rm /tmp/tmp.OAhR7HjJfu /tmp/tmp.4DhrhSEfuP + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.No3ZQj8nPK ++ mktemp + local LAST_ERR=/tmp/tmp.TFH2xfJifC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.No3ZQj8nPK perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-30577 namespace + cat /tmp/tmp.TFH2xfJifC + rm /tmp/tmp.No3ZQj8nPK /tmp/tmp.TFH2xfJifC + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.L9tXhQAZ8O ++ mktemp + local LAST_ERR=/tmp/tmp.keGkZSGrYI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.L9tXhQAZ8O deployment.apps "pxc-client" deleted from smart-update2-30577 namespace + cat /tmp/tmp.keGkZSGrYI + rm /tmp/tmp.L9tXhQAZ8O /tmp/tmp.keGkZSGrYI + return 0 + sleep 30 + desc 'Disabling telemetry on the operator level' + set +o xtrace ----------------------------------------------------------------------------------- Disabling telemetry on the operator level ----------------------------------------------------------------------------------- + kubectl_bin delete pod -l run=version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.JCB4OFFfjO ++ mktemp + local LAST_ERR=/tmp/tmp.vwDeKfwww2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JCB4OFFfjO pod "version-service-cr-76d4659869-n9d8d" deleted from smart-update2-30577 namespace + cat /tmp/tmp.vwDeKfwww2 + rm /tmp/tmp.JCB4OFFfjO /tmp/tmp.vwDeKfwww2 + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.BhFPVd6teG ++ mktemp + local LAST_ERR=/tmp/tmp.uoDQZMaFV0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BhFPVd6teG pod "version-service-559c4df55c-r9jqm" deleted from smart-update2-30577 namespace + cat /tmp/tmp.uoDQZMaFV0 + rm /tmp/tmp.BhFPVd6teG /tmp/tmp.uoDQZMaFV0 + return 0 + kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + kubectl_bin apply -n pxc-operator -f - + yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.1wUvQ52zEo + local LAST_OUT=/tmp/tmp.mG9PykQlxs ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.r8SAQ5Z4fV + local exit_status=0 + local LAST_ERR=/tmp/tmp.kR7W1AtdXN + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n pxc-operator -f - + for i in '$(seq 0 2)' + set +e + kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1wUvQ52zEo + cat /tmp/tmp.r8SAQ5Z4fV + rm /tmp/tmp.1wUvQ52zEo /tmp/tmp.r8SAQ5Z4fV + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mG9PykQlxs deployment.apps/percona-xtradb-cluster-operator configured + cat /tmp/tmp.kR7W1AtdXN + rm /tmp/tmp.mG9PykQlxs /tmp/tmp.kR7W1AtdXN + return 0 + sleep 30 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.4w1Usun1mM +++ mktemp ++ local LAST_ERR=/tmp/tmp.L5eq2J470r ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4w1Usun1mM ++ cat /tmp/tmp.L5eq2J470r ++ rm /tmp/tmp.4w1Usun1mM /tmp/tmp.L5eq2J470r ++ return 0 + wait_pod percona-xtradb-cluster-operator-7c6fd7c9f9-mb959 480 pxc-operator + local pod=percona-xtradb-cluster-operator-7c6fd7c9f9-mb959 + local max_retry=480 + local ns=pxc-operator ++ grep -E '^(pxc|proxysql)$' ++ echo percona-xtradb-cluster-operator-7c6fd7c9f9-mb959 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-7c6fd7c9f9-mb959 condition met E0702 04:30:31.750713 28013 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-7c6fd7c9f9-mb959&resourceVersion=1782966630830200000&timeoutSeconds=305&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-7c6fd7c9f9-mb959 to become Ready.Ok + check_telemetry_transfer http://version-service-cr.smart-update2-30577.svc.cluster.local:11000 8.0-recommended disabled + local cr_vs_uri=http://version-service-cr.smart-update2-30577.svc.cluster.local:11000 + local cr_vs_channel=8.0-recommended + local telemetry_state=disabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.p3lLpYONtb ++ mktemp + local LAST_ERR=/tmp/tmp.WQeMGDmuTo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.p3lLpYONtb deployment.apps/pxc-client created + cat /tmp/tmp.WQeMGDmuTo + rm /tmp/tmp.p3lLpYONtb /tmp/tmp.WQeMGDmuTo + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AGxpyuxyQi ++ mktemp + local LAST_ERR=/tmp/tmp.pTckgCcLTq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AGxpyuxyQi secret/minimal-cluster unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.pTckgCcLTq + rm /tmp/tmp.AGxpyuxyQi /tmp/tmp.pTckgCcLTq + return 0 + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858"' + kubectl_bin apply -f - + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.41-32.1"' + yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"' + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-30577.svc.cluster.local:11000"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/cr-minimal.yaml + yq eval '.spec.upgradeOptions.apply="8.0-recommended"' ++ mktemp + local LAST_OUT=/tmp/tmp.DeNhAon9by + yq eval '.spec.crVersion="9.9.9"' ++ mktemp + local LAST_ERR=/tmp/tmp.2IhVG4oNaG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DeNhAon9by perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.2IhVG4oNaG + rm /tmp/tmp.DeNhAon9by /tmp/tmp.2IhVG4oNaG + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-pxc ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-pxc ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo minimal-cluster-pxc-0 + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.1Aki5hiJsp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lupg5QBxxF +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.1Aki5hiJsp +++ cat /tmp/tmp.lupg5QBxxF +++ rm /tmp/tmp.1Aki5hiJsp /tmp/tmp.lupg5QBxxF +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-haproxy ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-haproxy ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met E0702 04:32:45.125269 14487 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dminimal-cluster-haproxy-0&resourceVersion=1782966760976189000&timeoutSeconds=570&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vVs9PDVJ3i +++ mktemp ++ local LAST_ERR=/tmp/tmp.wlvuzHdZU9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vVs9PDVJ3i ++ cat /tmp/tmp.wlvuzHdZU9 ++ rm /tmp/tmp.vVs9PDVJ3i /tmp/tmp.wlvuzHdZU9 ++ return 0 + client_pod=pxc-client-67fc4995bb-l7hlh + wait_pod pxc-client-67fc4995bb-l7hlh + local pod=pxc-client-67fc4995bb-l7hlh + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-l7hlh ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-l7hlh condition met E0702 04:32:52.826045 15623 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-l7hlh&resourceVersion=1782966771119964000&timeoutSeconds=404&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-l7hlh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.x5z2CaI276 +++ mktemp ++ local LAST_ERR=/tmp/tmp.3oPNFDttjs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.x5z2CaI276 ++ cat /tmp/tmp.3oPNFDttjs ++ rm /tmp/tmp.x5z2CaI276 /tmp/tmp.3oPNFDttjs ++ return 0 + client_pod=pxc-client-67fc4995bb-l7hlh + wait_pod pxc-client-67fc4995bb-l7hlh + local pod=pxc-client-67fc4995bb-l7hlh + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-l7hlh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-l7hlh condition met E0702 04:33:05.369531 17350 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-l7hlh&resourceVersion=1782966783622777000&timeoutSeconds=471&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-l7hlh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + jq 'del(."grpc.time_ms")' + grep -E 'server request payload|unary call' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.start_time")' + grep -Eo '\{.*\}' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-cr-76d4659869-j5mkj ++ mktemp + local LAST_OUT=/tmp/tmp.QbqVwkKwDT ++ mktemp + local LAST_ERR=/tmp/tmp.15z8x28N7k + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-76d4659869-j5mkj + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QbqVwkKwDT + cat /tmp/tmp.15z8x28N7k + rm /tmp/tmp.QbqVwkKwDT /tmp/tmp.15z8x28N7k + return 0 + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.start_time")' + grep -Eo '\{.*\}' + jq 'del(."grpc.request.content".msg.customResourceUid)' + grep -E 'server request payload|unary call' + jq 'del(."grpc.request.content".msg.kubeVersion)' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-559c4df55c-wpdvc ++ mktemp + local LAST_OUT=/tmp/tmp.hAoTg0dQXo ++ mktemp + local LAST_ERR=/tmp/tmp.bNxqPzHtmK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-559c4df55c-wpdvc + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hAoTg0dQXo + cat /tmp/tmp.bNxqPzHtmK + rm /tmp/tmp.hAoTg0dQXo /tmp/tmp.bNxqPzHtmK + return 0 + local telemetry_log_file=disabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' 8.0-recommended == disabled -a disabled == enabled ']' + local image_prefix=8.0 + image_prefix=8.0 + local telemetry_cr_log_file=disabled_telemetry.version-service-cr-8.0-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + [[ 8.0-recommended == \8\.\0\-\r\e\c\o\m\m\e\n\d\e\d ]] + '[' disabled == disabled ']' + desc 'cr VS should have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- cr VS should have telemetry ----------------------------------------------------------------------------------- + diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.0-cw.log.json /dev/fd/63 ++ grep -f /tmp/tmp.rCF6hRfwxh/disabled_telemetry.version-service-cr.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.0-cw.log.json + desc 'operator VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.rCF6hRfwxh/disabled_telemetry.version-service.log.json ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' 8.0-recommended == disabled -a disabled == disabled ']' + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.B8y6zfX3uf ++ mktemp + local LAST_ERR=/tmp/tmp.K7BGDJxX9B + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.B8y6zfX3uf perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.K7BGDJxX9B + rm /tmp/tmp.B8y6zfX3uf /tmp/tmp.K7BGDJxX9B + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.GnbzHRUq6v +++ mktemp ++ local LAST_ERR=/tmp/tmp.S9dWKNqu72 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GnbzHRUq6v ++ cat /tmp/tmp.S9dWKNqu72 ++ rm /tmp/tmp.GnbzHRUq6v /tmp/tmp.S9dWKNqu72 ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-7c6fd7c9f9-mb959 ++ mktemp + local LAST_OUT=/tmp/tmp.RIFBMHXAGS ++ mktemp + local LAST_ERR=/tmp/tmp.BZtcXRSMyU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-7c6fd7c9f9-mb959 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RIFBMHXAGS pod "percona-xtradb-cluster-operator-7c6fd7c9f9-mb959" deleted from pxc-operator namespace + cat /tmp/tmp.BZtcXRSMyU + rm /tmp/tmp.RIFBMHXAGS /tmp/tmp.BZtcXRSMyU + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.dVlNycpyas ++ mktemp + local LAST_ERR=/tmp/tmp.f8EOQqlnhM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dVlNycpyas perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-30577 namespace + cat /tmp/tmp.f8EOQqlnhM + rm /tmp/tmp.dVlNycpyas /tmp/tmp.f8EOQqlnhM + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.xryy2ZWeL9 ++ mktemp + local LAST_ERR=/tmp/tmp.82THxirIyb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xryy2ZWeL9 deployment.apps "pxc-client" deleted from smart-update2-30577 namespace + cat /tmp/tmp.82THxirIyb + rm /tmp/tmp.xryy2ZWeL9 /tmp/tmp.82THxirIyb + return 0 + sleep 30 + kubectl_bin delete pod -l run=version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.ro4jFvGpWn ++ mktemp + local LAST_ERR=/tmp/tmp.aLU9PfTRG2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ro4jFvGpWn pod "version-service-cr-76d4659869-j5mkj" deleted from smart-update2-30577 namespace + cat /tmp/tmp.aLU9PfTRG2 + rm /tmp/tmp.ro4jFvGpWn /tmp/tmp.aLU9PfTRG2 + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.39TSqKuQrH ++ mktemp + local LAST_ERR=/tmp/tmp.xTA75fz3BO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.39TSqKuQrH pod "version-service-559c4df55c-wpdvc" deleted from smart-update2-30577 namespace + cat /tmp/tmp.xTA75fz3BO + rm /tmp/tmp.39TSqKuQrH /tmp/tmp.xTA75fz3BO + return 0 + check_telemetry_transfer http://version-service-cr.smart-update2-30577.svc.cluster.local:11000 disabled disabled + local cr_vs_uri=http://version-service-cr.smart-update2-30577.svc.cluster.local:11000 + local cr_vs_channel=disabled + local telemetry_state=disabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Ay9rLvaIyY ++ mktemp + local LAST_ERR=/tmp/tmp.rGjVbPFPtm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Ay9rLvaIyY deployment.apps/pxc-client created + cat /tmp/tmp.rGjVbPFPtm + rm /tmp/tmp.Ay9rLvaIyY /tmp/tmp.rGjVbPFPtm + return 0 + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.HSoz8Lhs9W ++ mktemp + local LAST_ERR=/tmp/tmp.YvvHsffTN6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HSoz8Lhs9W secret/minimal-cluster unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.YvvHsffTN6 + rm /tmp/tmp.HSoz8Lhs9W /tmp/tmp.YvvHsffTN6 + return 0 + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-30577.svc.cluster.local:11000"' + kubectl_bin apply -f - + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.41-32.1"' + yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"' + yq eval '.spec.crVersion="9.9.9"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/cr-minimal.yaml + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858"' + yq eval '.spec.upgradeOptions.apply="disabled"' ++ mktemp + local LAST_OUT=/tmp/tmp.TnEJJ6wfJU ++ mktemp + local LAST_ERR=/tmp/tmp.E82wuGBVpN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TnEJJ6wfJU perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.E82wuGBVpN + rm /tmp/tmp.TnEJJ6wfJU /tmp/tmp.E82wuGBVpN + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-pxc ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-pxc ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo minimal-cluster-pxc-0 + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Q4vDNBWJeG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bDe6CEHQzL +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Q4vDNBWJeG +++ cat /tmp/tmp.bDe6CEHQzL +++ rm /tmp/tmp.Q4vDNBWJeG /tmp/tmp.bDe6CEHQzL +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-haproxy ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-haproxy ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met E0702 04:36:29.697739 13626 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dminimal-cluster-haproxy-0&resourceVersion=1782966986720146000&timeoutSeconds=353&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oiKGYJiWYB +++ mktemp ++ local LAST_ERR=/tmp/tmp.8aPWYlUr8v ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oiKGYJiWYB ++ cat /tmp/tmp.8aPWYlUr8v ++ rm /tmp/tmp.oiKGYJiWYB /tmp/tmp.8aPWYlUr8v ++ return 0 + client_pod=pxc-client-67fc4995bb-z27hs + wait_pod pxc-client-67fc4995bb-z27hs + local pod=pxc-client-67fc4995bb-z27hs + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-z27hs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-z27hs condition met E0702 04:36:41.917205 15387 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-z27hs&resourceVersion=1782966998792529000&timeoutSeconds=574&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-z27hs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lQT8dgZSL7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.AeKgolMkr7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lQT8dgZSL7 ++ cat /tmp/tmp.AeKgolMkr7 ++ rm /tmp/tmp.lQT8dgZSL7 /tmp/tmp.AeKgolMkr7 ++ return 0 + client_pod=pxc-client-67fc4995bb-z27hs + wait_pod pxc-client-67fc4995bb-z27hs + local pod=pxc-client-67fc4995bb-z27hs + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-z27hs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-z27hs condition met E0702 04:37:00.121011 18170 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-z27hs&resourceVersion=1782967017584032000&timeoutSeconds=555&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-z27hs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + grep -Eo '\{.*\}' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.request.content".msg.kubeVersion)' + grep -E 'server request payload|unary call' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.time_ms")' + kubectl_bin logs version-service-cr-76d4659869-xbbl7 ++ mktemp + local LAST_OUT=/tmp/tmp.lh6sdF0Qux ++ mktemp + local LAST_ERR=/tmp/tmp.PFlaoOkm57 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-76d4659869-xbbl7 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lh6sdF0Qux + cat /tmp/tmp.PFlaoOkm57 + rm /tmp/tmp.lh6sdF0Qux /tmp/tmp.PFlaoOkm57 + return 0 + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + grep -E 'server request payload|unary call' + jq 'del(."grpc.request.content".msg.kubeVersion)' + grep -Eo '\{.*\}' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-559c4df55c-6nr8n ++ mktemp + local LAST_OUT=/tmp/tmp.9djt7XG4yH ++ mktemp + local LAST_ERR=/tmp/tmp.Ywlul0eNVO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-559c4df55c-6nr8n + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9djt7XG4yH + cat /tmp/tmp.Ywlul0eNVO + rm /tmp/tmp.9djt7XG4yH /tmp/tmp.Ywlul0eNVO + return 0 + local telemetry_log_file=disabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' disabled == disabled -a disabled == enabled ']' + local image_prefix=disabled + image_prefix=disabled + local telemetry_cr_log_file=disabled_telemetry.version-service-cr-disabled-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + [[ disabled == \d\i\s\a\b\l\e\d\-\r\e\c\o\m\m\e\n\d\e\d ]] + [[ disabled == \d\i\s\a\b\l\e\d\-\l\a\t\e\s\t ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' disabled == disabled -a disabled == disabled ']' + desc 'CR VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- CR VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.rCF6hRfwxh/disabled_telemetry.version-service-cr.log.json ]] + desc 'operator VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.rCF6hRfwxh/disabled_telemetry.version-service.log.json ]] + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.lUmkqnfmzK ++ mktemp + local LAST_ERR=/tmp/tmp.vlXQ6nSBRM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lUmkqnfmzK perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.vlXQ6nSBRM + rm /tmp/tmp.lUmkqnfmzK /tmp/tmp.vlXQ6nSBRM + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.97OOVTlPia ++ head -1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.n0thdwyxGi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.97OOVTlPia ++ cat /tmp/tmp.n0thdwyxGi ++ rm /tmp/tmp.97OOVTlPia /tmp/tmp.n0thdwyxGi ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-7c6fd7c9f9-k7t4n ++ mktemp + local LAST_OUT=/tmp/tmp.tXrLytXxch ++ mktemp + local LAST_ERR=/tmp/tmp.6bvFyVHGsD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-7c6fd7c9f9-k7t4n + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tXrLytXxch pod "percona-xtradb-cluster-operator-7c6fd7c9f9-k7t4n" deleted from pxc-operator namespace + cat /tmp/tmp.6bvFyVHGsD + rm /tmp/tmp.tXrLytXxch /tmp/tmp.6bvFyVHGsD + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.bVZDLaWkrQ ++ mktemp + local LAST_ERR=/tmp/tmp.2RfegW9paS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bVZDLaWkrQ perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-30577 namespace + cat /tmp/tmp.2RfegW9paS + rm /tmp/tmp.bVZDLaWkrQ /tmp/tmp.2RfegW9paS + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.8totwlbEqh ++ mktemp + local LAST_ERR=/tmp/tmp.9WUHAR85sH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8totwlbEqh deployment.apps "pxc-client" deleted from smart-update2-30577 namespace + cat /tmp/tmp.9WUHAR85sH + rm /tmp/tmp.8totwlbEqh /tmp/tmp.9WUHAR85sH + return 0 + sleep 30 + kubectl_bin delete deployment version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.JYiIE4IEAG ++ mktemp + local LAST_ERR=/tmp/tmp.ryhP0z7hE5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deployment version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JYiIE4IEAG deployment.apps "version-service-cr" deleted from smart-update2-30577 namespace + cat /tmp/tmp.ryhP0z7hE5 + rm /tmp/tmp.JYiIE4IEAG /tmp/tmp.ryhP0z7hE5 + return 0 + desc 'Telemetry testing finished' + set +o xtrace ----------------------------------------------------------------------------------- Telemetry testing finished ----------------------------------------------------------------------------------- + desc 'PXC cluster with version service offline' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster with version service offline ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/smart-update-version-service-unreachable.yml /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858"' /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + spinup_pxc smart-update /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.GXkkSHagGF ++ mktemp + local LAST_ERR=/tmp/tmp.KTqpRfoH1k + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GXkkSHagGF secret/my-cluster-secrets created secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.KTqpRfoH1k + rm /tmp/tmp.GXkkSHagGF /tmp/tmp.KTqpRfoH1k + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + local LAST_OUT=/tmp/tmp.788d6SdWGE + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-30577~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.Tu0HjMpYg8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.788d6SdWGE deployment.apps/pxc-client created + cat /tmp/tmp.Tu0HjMpYg8 + rm /tmp/tmp.788d6SdWGE /tmp/tmp.Tu0HjMpYg8 + return 0 + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + apply_config /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local config_file=/tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local pvc_name= + '[' -z '' ']' + cat_config /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml '' + local input_file=/tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local pvc_name= + kubectl_bin apply -f - + cat /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_OUT=/tmp/tmp.467g4X515h + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-30577~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + local LAST_ERR=/tmp/tmp.BiKBqio6bT + local exit_status=0 + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.467g4X515h perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.BiKBqio6bT + rm /tmp/tmp.467g4X515h /tmp/tmp.BiKBqio6bT + return 0 + desc 'waiting pods to be running for smart-update' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for smart-update ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sptLSwPrug ++++ mktemp +++ local LAST_ERR=/tmp/tmp.a2620ylEIc +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.sptLSwPrug +++ cat /tmp/tmp.a2620ylEIc +++ rm /tmp/tmp.sptLSwPrug /tmp/tmp.a2620ylEIc +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy ++ get_proxy_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XtA9gjfNGQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MSDKGRLt8s +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.XtA9gjfNGQ +++ cat /tmp/tmp.MSDKGRLt8s +++ rm /tmp/tmp.XtA9gjfNGQ /tmp/tmp.MSDKGRLt8s +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5MryDlV5HN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NiOGaobcET +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.5MryDlV5HN +++ cat /tmp/tmp.NiOGaobcET +++ rm /tmp/tmp.5MryDlV5HN /tmp/tmp.NiOGaobcET +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z smart-update-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for smart-update-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for smart-update-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-2 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + wait_for_running smart-update-haproxy 2 + local name=smart-update-haproxy + local size=2 + local last_pod + local max_retry=480 + [[ -z smart-update-haproxy ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for smart-update-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for smart-update-haproxy ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met E0702 04:43:07.821787 7542 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-0&resourceVersion=1782967384425460000&timeoutSeconds=524&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-1 480 + local pod=smart-update-haproxy-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-haproxy-1 + local container= + set +o xtrace pod/smart-update-haproxy-1 condition met E0702 04:43:15.293978 9088 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-1&resourceVersion=1782967394425577000&timeoutSeconds=340&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-1 to become Ready.Ok + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WxE0nVrnOk +++ mktemp ++ local LAST_ERR=/tmp/tmp.BehZW2oWMn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WxE0nVrnOk ++ cat /tmp/tmp.BehZW2oWMn ++ rm /tmp/tmp.WxE0nVrnOk /tmp/tmp.BehZW2oWMn ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.y81zTEbGK6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.LZ0qbz1oRq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.y81zTEbGK6 ++ cat /tmp/tmp.LZ0qbz1oRq ++ rm /tmp/tmp.y81zTEbGK6 /tmp/tmp.LZ0qbz1oRq ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.NJCJqpToN2 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Q25VbhYQ52 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.NJCJqpToN2 +++++ cat /tmp/tmp.Q25VbhYQ52 +++++ rm /tmp/tmp.NJCJqpToN2 /tmp/tmp.Q25VbhYQ52 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GmtbbkrZCT +++ mktemp ++ local LAST_ERR=/tmp/tmp.JoV4LFrVCf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GmtbbkrZCT ++ cat /tmp/tmp.JoV4LFrVCf ++ rm /tmp/tmp.GmtbbkrZCT /tmp/tmp.JoV4LFrVCf ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.H3KaXVBUri +++ mktemp ++ local LAST_ERR=/tmp/tmp.Wn1mesUNx4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.H3KaXVBUri ++ cat /tmp/tmp.Wn1mesUNx4 ++ rm /tmp/tmp.H3KaXVBUri /tmp/tmp.Wn1mesUNx4 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3O63sootof +++ mktemp ++ local LAST_ERR=/tmp/tmp.snLn6BiYgd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3O63sootof ++ cat /tmp/tmp.snLn6BiYgd ++ rm /tmp/tmp.3O63sootof /tmp/tmp.snLn6BiYgd ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:44:00.716492 15662 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782967438123387000&timeoutSeconds=463&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data smart-update 100500 no-proxy -P3306 + local cluster=smart-update + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.KrzhhzpfGg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2DesLcG0A1 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.KrzhhzpfGg +++ cat /tmp/tmp.2DesLcG0A1 +++ rm /tmp/tmp.KrzhhzpfGg /tmp/tmp.2DesLcG0A1 +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FhwHCPoQZw +++ mktemp ++ local LAST_ERR=/tmp/tmp.8sCKqZkYUh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FhwHCPoQZw ++ cat /tmp/tmp.8sCKqZkYUh ++ rm /tmp/tmp.FhwHCPoQZw /tmp/tmp.8sCKqZkYUh ++ return 0 + root_pass=root_password ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.IPFNnbuJbs ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RKKagL6aRX +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.IPFNnbuJbs +++ cat /tmp/tmp.RKKagL6aRX +++ rm /tmp/tmp.IPFNnbuJbs /tmp/tmp.RKKagL6aRX +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + proxy=smart-update-haproxy + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h smart-update-pxc-0.smart-update-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kO0CnAL58c +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z0PjqsViIn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kO0CnAL58c ++ cat /tmp/tmp.Z0PjqsViIn ++ rm /tmp/tmp.kO0CnAL58c /tmp/tmp.Z0PjqsViIn ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-7gzlp + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:44:16.742089 18013 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782967454182114000&timeoutSeconds=335&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data smart-update select-1 no-proxy -P3306 + local cluster=smart-update + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lmXyf6hxZf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MEh5wR5Whq +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.lmXyf6hxZf +++ cat /tmp/tmp.MEh5wR5Whq +++ rm /tmp/tmp.lmXyf6hxZf /tmp/tmp.MEh5wR5Whq +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WJWv5g5Fuf +++ mktemp ++ local LAST_ERR=/tmp/tmp.6TjFWtYn9m ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WJWv5g5Fuf ++ cat /tmp/tmp.6TjFWtYn9m ++ rm /tmp/tmp.WJWv5g5Fuf /tmp/tmp.6TjFWtYn9m ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.g1krTNob5v ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rFHEYzU6jI +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.g1krTNob5v +++ cat /tmp/tmp.rFHEYzU6jI +++ rm /tmp/tmp.g1krTNob5v /tmp/tmp.rFHEYzU6jI +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2nsstBczFn +++ mktemp ++ local LAST_ERR=/tmp/tmp.sDd7eTXLis ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2nsstBczFn ++ cat /tmp/tmp.sDd7eTXLis ++ rm /tmp/tmp.2nsstBczFn /tmp/tmp.sDd7eTXLis ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:45:07.349433 25684 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782967504324843000&timeoutSeconds=556&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UBKMcroFIn +++ mktemp ++ local LAST_ERR=/tmp/tmp.St6Ca6bOyS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UBKMcroFIn ++ cat /tmp/tmp.St6Ca6bOyS ++ rm /tmp/tmp.UBKMcroFIn /tmp/tmp.St6Ca6bOyS ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:45:23.511776 28054 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782967521304454000&timeoutSeconds=457&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5ybWTQImvI +++ mktemp ++ local LAST_ERR=/tmp/tmp.F4DtS1rcuQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5ybWTQImvI ++ cat /tmp/tmp.F4DtS1rcuQ ++ rm /tmp/tmp.5ybWTQImvI /tmp/tmp.F4DtS1rcuQ ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-7gzlp + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:45:38.809369 30275 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782967535943442000&timeoutSeconds=395&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + grep -E -o 'early-plugin-load=keyring_\w+.so' + kubectl exec smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + return 1 + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dmmDbqMXEa +++ mktemp ++ local LAST_ERR=/tmp/tmp.qbH7D9jp41 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dmmDbqMXEa ++ cat /tmp/tmp.qbH7D9jp41 ++ rm /tmp/tmp.dmmDbqMXEa /tmp/tmp.qbH7D9jp41 ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w5WGD1s0VM +++ mktemp ++ local LAST_ERR=/tmp/tmp.pEiiQF84vM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.w5WGD1s0VM ++ cat /tmp/tmp.pEiiQF84vM ++ rm /tmp/tmp.w5WGD1s0VM /tmp/tmp.pEiiQF84vM ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.5uYLzfhblS ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.DQNpKWORBj +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.5uYLzfhblS +++++ cat /tmp/tmp.DQNpKWORBj +++++ rm /tmp/tmp.5uYLzfhblS /tmp/tmp.DQNpKWORBj +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mtQtJCINXc +++ mktemp ++ local LAST_ERR=/tmp/tmp.I2NQFqOXZK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mtQtJCINXc ++ cat /tmp/tmp.I2NQFqOXZK ++ rm /tmp/tmp.mtQtJCINXc /tmp/tmp.I2NQFqOXZK ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ kubectl_bin get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eiRxTa8GPV +++ mktemp ++ local LAST_ERR=/tmp/tmp.lhZGbQkmjP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eiRxTa8GPV ++ cat /tmp/tmp.lhZGbQkmjP ++ rm /tmp/tmp.eiRxTa8GPV /tmp/tmp.lhZGbQkmjP ++ return 0 + [[ percona/percona-xtradb-cluster:8.0.41-32.1 != \p\e\r\c\o\n\a\/\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\:\8\.\0\.\4\1\-\3\2\.\1 ]] + desc 'PXC cluster update with recommended image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with recommended image by version service ----------------------------------------------------------------------------------- + vs_image=recommended ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kxj2fWad0m ++++ mktemp +++ local LAST_ERR=/tmp/tmp.viypEk46Jk +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.kxj2fWad0m +++ cat /tmp/tmp.viypEk46Jk +++ rm /tmp/tmp.kxj2fWad0m /tmp/tmp.viypEk46Jk +++ return 0 ++ client_pod=pxc-client-6666d5df5c-7gzlp ++ wait_pod pxc-client-6666d5df5c-7gzlp ++ local pod=pxc-client-6666d5df5c-7gzlp ++ local max_retry=480 ++ local ns= +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ grep -E '^(pxc|proxysql)$' +++ echo pxc-client-6666d5df5c-7gzlp ++ local container= ++ set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:46:16.656023 3544 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782967573054459000&timeoutSeconds=346&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.qoqG5eAFaf ++ mktemp + local LAST_ERR=/tmp/tmp.u6apAItT5u + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qoqG5eAFaf perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.u6apAItT5u + rm /tmp/tmp.qoqG5eAFaf /tmp/tmp.u6apAItT5u + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PCEFtfw2b7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.esNJuDyBXK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PCEFtfw2b7 ++ cat /tmp/tmp.esNJuDyBXK ++ rm /tmp/tmp.PCEFtfw2b7 /tmp/tmp.esNJuDyBXK ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hxIjrJ5dUl +++ mktemp ++ local LAST_ERR=/tmp/tmp.qN8UHeLYWq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hxIjrJ5dUl ++ cat /tmp/tmp.qN8UHeLYWq ++ rm /tmp/tmp.hxIjrJ5dUl /tmp/tmp.qN8UHeLYWq ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.0KQbnR6k5f ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.lpPS1PMVMi +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.0KQbnR6k5f +++++ cat /tmp/tmp.lpPS1PMVMi +++++ rm /tmp/tmp.0KQbnR6k5f /tmp/tmp.lpPS1PMVMi +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1EHvZyQ3BL +++ mktemp ++ local LAST_ERR=/tmp/tmp.LzAlflN3AI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1EHvZyQ3BL ++ cat /tmp/tmp.LzAlflN3AI ++ rm /tmp/tmp.1EHvZyQ3BL /tmp/tmp.LzAlflN3AI ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z0seAq9mtM +++ mktemp ++ local LAST_ERR=/tmp/tmp.5yPHH53EIo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.z0seAq9mtM ++ cat /tmp/tmp.5yPHH53EIo ++ rm /tmp/tmp.z0seAq9mtM /tmp/tmp.5yPHH53EIo ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6666d5df5c-7gzlp ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:47:57.548859 18678 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782967673962927000&timeoutSeconds=368&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BGeMfTYFEc +++ mktemp ++ local LAST_ERR=/tmp/tmp.B467r39bAr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BGeMfTYFEc ++ cat /tmp/tmp.B467r39bAr ++ rm /tmp/tmp.BGeMfTYFEc /tmp/tmp.B467r39bAr ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:48:11.057009 20875 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782967688800256000&timeoutSeconds=356&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0LBOQGFDlE +++ mktemp ++ local LAST_ERR=/tmp/tmp.EbN8gYD3pM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0LBOQGFDlE ++ cat /tmp/tmp.EbN8gYD3pM ++ rm /tmp/tmp.0LBOQGFDlE /tmp/tmp.EbN8gYD3pM ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:48:22.205278 22576 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782967699742396000&timeoutSeconds=508&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + kubectl_bin delete -f /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.sOkGv3C2C6 ++ mktemp + local LAST_ERR=/tmp/tmp.bzah8XtdwR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sOkGv3C2C6 perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-30577 namespace + cat /tmp/tmp.bzah8XtdwR + rm /tmp/tmp.sOkGv3C2C6 /tmp/tmp.bzah8XtdwR + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.SgT0uqFql7 ++ mktemp + local LAST_ERR=/tmp/tmp.Rs6rDoREFV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SgT0uqFql7 persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-30577 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-30577 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-30577 namespace + cat /tmp/tmp.Rs6rDoREFV + rm /tmp/tmp.SgT0uqFql7 /tmp/tmp.Rs6rDoREFV + return 0 + desc 'PXC cluster update with the latest image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with the latest image by version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.boZy2046Gv ++ mktemp + local LAST_ERR=/tmp/tmp.FdrA8JljeA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.boZy2046Gv secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.FdrA8JljeA + rm /tmp/tmp.boZy2046Gv /tmp/tmp.FdrA8JljeA + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml '' + kubectl_bin apply -f - + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-30577~ + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_OUT=/tmp/tmp.hiDNuWTvpb + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' ++ mktemp + local LAST_ERR=/tmp/tmp.eBMYZDO4Nq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hiDNuWTvpb deployment.apps/pxc-client unchanged + cat /tmp/tmp.eBMYZDO4Nq + rm /tmp/tmp.hiDNuWTvpb /tmp/tmp.eBMYZDO4Nq + return 0 + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + apply_config /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local config_file=/tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local pvc_name= + '[' -z '' ']' + cat_config /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml '' + local input_file=/tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local pvc_name= + kubectl_bin apply -f - + cat /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-30577~ ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + local LAST_OUT=/tmp/tmp.123qN8DM3W + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' ++ mktemp + local LAST_ERR=/tmp/tmp.0GCLHdZcHP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.123qN8DM3W perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.0GCLHdZcHP + rm /tmp/tmp.123qN8DM3W /tmp/tmp.0GCLHdZcHP + return 0 + desc 'waiting pods to be running for smart-update' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for smart-update ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LFjVxO2EHB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.U1uy4uRLcu +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.LFjVxO2EHB +++ cat /tmp/tmp.U1uy4uRLcu +++ rm /tmp/tmp.LFjVxO2EHB /tmp/tmp.U1uy4uRLcu +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy ++ get_proxy_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5EhJg1Lk12 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JAQC0ywdFs +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.5EhJg1Lk12 +++ cat /tmp/tmp.JAQC0ywdFs +++ rm /tmp/tmp.5EhJg1Lk12 /tmp/tmp.JAQC0ywdFs +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LfwsLOiDFw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iouSjRgWox +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.LfwsLOiDFw +++ cat /tmp/tmp.iouSjRgWox +++ rm /tmp/tmp.LfwsLOiDFw /tmp/tmp.iouSjRgWox +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z smart-update-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for smart-update-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for smart-update-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + wait_for_running smart-update-haproxy 2 + local name=smart-update-haproxy + local size=2 + local last_pod + local max_retry=480 + [[ -z smart-update-haproxy ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for smart-update-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for smart-update-haproxy ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-haproxy-0 ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met E0702 04:53:39.006718 6099 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-0&resourceVersion=1782968018040113000&timeoutSeconds=341&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-1 480 + local pod=smart-update-haproxy-1 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-1 condition met E0702 04:53:50.183091 7955 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-1&resourceVersion=1782968028040177000&timeoutSeconds=507&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-1 to become Ready.Ok + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cZvdGDFw7j +++ mktemp ++ local LAST_ERR=/tmp/tmp.vyzIITfopx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cZvdGDFw7j ++ cat /tmp/tmp.vyzIITfopx ++ rm /tmp/tmp.cZvdGDFw7j /tmp/tmp.vyzIITfopx ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oE36Y2BIcr +++ mktemp ++ local LAST_ERR=/tmp/tmp.qqzwvh863x ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oE36Y2BIcr ++ cat /tmp/tmp.qqzwvh863x ++ rm /tmp/tmp.oE36Y2BIcr /tmp/tmp.qqzwvh863x ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.B2uC8EhuqS ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.jGBqyZU5cg +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.B2uC8EhuqS +++++ cat /tmp/tmp.jGBqyZU5cg +++++ rm /tmp/tmp.B2uC8EhuqS /tmp/tmp.jGBqyZU5cg +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aHmuUeD5eK +++ mktemp ++ local LAST_ERR=/tmp/tmp.5dNxpVRoEt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aHmuUeD5eK ++ cat /tmp/tmp.5dNxpVRoEt ++ rm /tmp/tmp.aHmuUeD5eK /tmp/tmp.5dNxpVRoEt ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1ZqRDtiRwP +++ mktemp ++ local LAST_ERR=/tmp/tmp.A8oznjJPWE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1ZqRDtiRwP ++ cat /tmp/tmp.A8oznjJPWE ++ rm /tmp/tmp.1ZqRDtiRwP /tmp/tmp.A8oznjJPWE ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lV8JndtlY2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.qeIXHFPERV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lV8JndtlY2 ++ cat /tmp/tmp.qeIXHFPERV ++ rm /tmp/tmp.lV8JndtlY2 /tmp/tmp.qeIXHFPERV ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:54:33.671011 14608 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968071646741000&timeoutSeconds=579&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data smart-update 100500 no-proxy -P3306 + local cluster=smart-update + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TVsiU64fb1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Gd6o0oIKKL +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.TVsiU64fb1 +++ cat /tmp/tmp.Gd6o0oIKKL +++ rm /tmp/tmp.TVsiU64fb1 /tmp/tmp.Gd6o0oIKKL +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.epTgCStUho +++ mktemp ++ local LAST_ERR=/tmp/tmp.faYHcRmjFI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.epTgCStUho ++ cat /tmp/tmp.faYHcRmjFI ++ rm /tmp/tmp.epTgCStUho /tmp/tmp.faYHcRmjFI ++ return 0 + root_pass=root_password ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3YKbzToohp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rveHUHkEWB +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.3YKbzToohp +++ cat /tmp/tmp.rveHUHkEWB +++ rm /tmp/tmp.3YKbzToohp /tmp/tmp.rveHUHkEWB +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + proxy=smart-update-haproxy + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h smart-update-pxc-0.smart-update-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MzjaxkHqfn +++ mktemp ++ local LAST_ERR=/tmp/tmp.3rA7Sxooh6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MzjaxkHqfn ++ cat /tmp/tmp.3rA7Sxooh6 ++ rm /tmp/tmp.MzjaxkHqfn /tmp/tmp.3rA7Sxooh6 ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:54:56.743538 18004 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968093009016000&timeoutSeconds=327&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data smart-update select-1 no-proxy -P3306 + local cluster=smart-update + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bwHnUMt1rw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.98tANIBnlt +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.bwHnUMt1rw +++ cat /tmp/tmp.98tANIBnlt +++ rm /tmp/tmp.bwHnUMt1rw /tmp/tmp.98tANIBnlt +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eyd9MEZ377 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4uxigAO5Ju ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eyd9MEZ377 ++ cat /tmp/tmp.4uxigAO5Ju ++ rm /tmp/tmp.eyd9MEZ377 /tmp/tmp.4uxigAO5Ju ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2BohSRupPk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kvlbafmFLF +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.2BohSRupPk +++ cat /tmp/tmp.kvlbafmFLF +++ rm /tmp/tmp.2BohSRupPk /tmp/tmp.kvlbafmFLF +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.arQ4pGquM0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.7VazdZEIXe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.arQ4pGquM0 ++ cat /tmp/tmp.7VazdZEIXe ++ rm /tmp/tmp.arQ4pGquM0 /tmp/tmp.7VazdZEIXe ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:55:48.912591 26099 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968145617332000&timeoutSeconds=364&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ac0nZVyn18 +++ mktemp ++ local LAST_ERR=/tmp/tmp.dCGBMdpzHx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ac0nZVyn18 ++ cat /tmp/tmp.dCGBMdpzHx ++ rm /tmp/tmp.Ac0nZVyn18 /tmp/tmp.dCGBMdpzHx ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:56:01.089583 28026 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968158471545000&timeoutSeconds=463&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.soc02Iv451 +++ mktemp ++ local LAST_ERR=/tmp/tmp.UM3Jr4TNGb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.soc02Iv451 ++ cat /tmp/tmp.UM3Jr4TNGb ++ rm /tmp/tmp.soc02Iv451 /tmp/tmp.UM3Jr4TNGb ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:56:17.465206 30431 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968174019325000&timeoutSeconds=362&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + grep -E -o 'early-plugin-load=keyring_\w+.so' + kubectl exec smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + return 1 + vs_image=latest ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.w6ZeuEj1lo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9COHE9cjzW +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.w6ZeuEj1lo +++ cat /tmp/tmp.9COHE9cjzW +++ rm /tmp/tmp.w6ZeuEj1lo /tmp/tmp.9COHE9cjzW +++ return 0 ++ client_pod=pxc-client-6666d5df5c-7gzlp ++ wait_pod pxc-client-6666d5df5c-7gzlp ++ local pod=pxc-client-6666d5df5c-7gzlp ++ local max_retry=480 ++ local ns= +++ grep -E '^(pxc|proxysql)$' +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ echo pxc-client-6666d5df5c-7gzlp ++ local container= ++ set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:56:37.151895 804 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968194169448000&timeoutSeconds=330&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.SpZGsiBnOd ++ mktemp + local LAST_ERR=/tmp/tmp.RXqfX72Vzf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SpZGsiBnOd perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.RXqfX72Vzf + rm /tmp/tmp.SpZGsiBnOd /tmp/tmp.RXqfX72Vzf + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XZ9P1ld19y +++ mktemp ++ local LAST_ERR=/tmp/tmp.ez3QoAHqgh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XZ9P1ld19y ++ cat /tmp/tmp.ez3QoAHqgh ++ rm /tmp/tmp.XZ9P1ld19y /tmp/tmp.ez3QoAHqgh ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OehuGdXoxt +++ mktemp ++ local LAST_ERR=/tmp/tmp.TsJ3ugwNgk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OehuGdXoxt ++ cat /tmp/tmp.TsJ3ugwNgk ++ rm /tmp/tmp.OehuGdXoxt /tmp/tmp.TsJ3ugwNgk ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Yuuzn22q50 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.7XTo3xkG3J +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Yuuzn22q50 +++++ cat /tmp/tmp.7XTo3xkG3J +++++ rm /tmp/tmp.Yuuzn22q50 /tmp/tmp.7XTo3xkG3J +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YKX53Z2aTh +++ mktemp ++ local LAST_ERR=/tmp/tmp.AaOZTggHOE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YKX53Z2aTh ++ cat /tmp/tmp.AaOZTggHOE ++ rm /tmp/tmp.YKX53Z2aTh /tmp/tmp.AaOZTggHOE ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KR3QmSeljp +++ mktemp ++ local LAST_ERR=/tmp/tmp.8wZjyFpydS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KR3QmSeljp ++ cat /tmp/tmp.8wZjyFpydS ++ rm /tmp/tmp.KR3QmSeljp /tmp/tmp.8wZjyFpydS ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:58:01.534310 13397 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968279859110000&timeoutSeconds=446&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Gmar4Kv90p +++ mktemp ++ local LAST_ERR=/tmp/tmp.OVnsCtgC3X ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Gmar4Kv90p ++ cat /tmp/tmp.OVnsCtgC3X ++ rm /tmp/tmp.Gmar4Kv90p /tmp/tmp.OVnsCtgC3X ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:58:13.125970 15038 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968290043450000&timeoutSeconds=390&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bAm2YyMkom +++ mktemp ++ local LAST_ERR=/tmp/tmp.9csijnCT3Q ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bAm2YyMkom ++ cat /tmp/tmp.9csijnCT3Q ++ rm /tmp/tmp.bAm2YyMkom /tmp/tmp.9csijnCT3Q ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 04:58:33.947707 17991 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968309694772000&timeoutSeconds=446&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + kubectl_bin delete -f /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.k7y8hNUuCx ++ mktemp + local LAST_ERR=/tmp/tmp.EPEsfGbUmD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.k7y8hNUuCx perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-30577 namespace + cat /tmp/tmp.EPEsfGbUmD + rm /tmp/tmp.k7y8hNUuCx /tmp/tmp.EPEsfGbUmD + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.ex69erSrUL ++ mktemp + local LAST_ERR=/tmp/tmp.JPA2tE0aAv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ex69erSrUL persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-30577 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-30577 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-30577 namespace + cat /tmp/tmp.JPA2tE0aAv + rm /tmp/tmp.ex69erSrUL /tmp/tmp.JPA2tE0aAv + return 0 + desc 'PXC cluster update with explicitly specified image inside version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with explicitly specified image inside version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.3gjLtLHmIN ++ mktemp + local LAST_ERR=/tmp/tmp.XPF0VR9n6v + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3gjLtLHmIN secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.XPF0VR9n6v + rm /tmp/tmp.3gjLtLHmIN /tmp/tmp.XPF0VR9n6v + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-30577~ + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.ptTfBFVBqq + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' ++ mktemp + local LAST_ERR=/tmp/tmp.HhcKI2Pni5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ptTfBFVBqq deployment.apps/pxc-client unchanged + cat /tmp/tmp.HhcKI2Pni5 + rm /tmp/tmp.ptTfBFVBqq /tmp/tmp.HhcKI2Pni5 + return 0 + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + apply_config /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local config_file=/tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local pvc_name= + '[' -z '' ']' + cat_config /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml '' + kubectl_bin apply -f - + local input_file=/tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + local LAST_OUT=/tmp/tmp.CjITVJop56 ++ mktemp + local LAST_ERR=/tmp/tmp.4IUzzLpWl9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + cat /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-30577~ + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CjITVJop56 perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.4IUzzLpWl9 + rm /tmp/tmp.CjITVJop56 /tmp/tmp.4IUzzLpWl9 + return 0 + desc 'waiting pods to be running for smart-update' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for smart-update ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.I3rUJJJkML ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OSDCzqgZbt +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.I3rUJJJkML +++ cat /tmp/tmp.OSDCzqgZbt +++ rm /tmp/tmp.I3rUJJJkML /tmp/tmp.OSDCzqgZbt +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy ++ get_proxy_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.p2ft7gb8Bv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ATd0bVA8xm +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.p2ft7gb8Bv +++ cat /tmp/tmp.ATd0bVA8xm +++ rm /tmp/tmp.p2ft7gb8Bv /tmp/tmp.ATd0bVA8xm +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.eMOXFbq1S5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GcFrX570IV +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.eMOXFbq1S5 +++ cat /tmp/tmp.GcFrX570IV +++ rm /tmp/tmp.eMOXFbq1S5 /tmp/tmp.GcFrX570IV +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z smart-update-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for smart-update-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for smart-update-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-pxc-0 + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-2 + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + wait_for_running smart-update-haproxy 2 + local name=smart-update-haproxy + local size=2 + local last_pod + local max_retry=480 + [[ -z smart-update-haproxy ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for smart-update-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for smart-update-haproxy ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-haproxy-0 + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met E0702 05:03:57.014630 27797 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-0&resourceVersion=1782968634380202000&timeoutSeconds=445&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-1 480 + local pod=smart-update-haproxy-1 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-1 condition met E0702 05:04:03.220791 28846 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-1&resourceVersion=1782968639380251000&timeoutSeconds=330&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-1 to become Ready.Ok + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AAsr9kiQ87 +++ mktemp ++ local LAST_ERR=/tmp/tmp.6QzoGwK0RA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AAsr9kiQ87 ++ cat /tmp/tmp.6QzoGwK0RA ++ rm /tmp/tmp.AAsr9kiQ87 /tmp/tmp.6QzoGwK0RA ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cjSDVKfacj +++ mktemp ++ local LAST_ERR=/tmp/tmp.g1xniynykd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cjSDVKfacj ++ cat /tmp/tmp.g1xniynykd ++ rm /tmp/tmp.cjSDVKfacj /tmp/tmp.g1xniynykd ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.R537g5ydaX ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.M7A4fhl1v8 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.R537g5ydaX +++++ cat /tmp/tmp.M7A4fhl1v8 +++++ rm /tmp/tmp.R537g5ydaX /tmp/tmp.M7A4fhl1v8 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.snyEIA2srm +++ mktemp ++ local LAST_ERR=/tmp/tmp.BEeLp8dpNw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.snyEIA2srm ++ cat /tmp/tmp.BEeLp8dpNw ++ rm /tmp/tmp.snyEIA2srm /tmp/tmp.BEeLp8dpNw ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.Oj05WDqbww +++ mktemp ++ local LAST_ERR=/tmp/tmp.NmAK4zgPl3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Oj05WDqbww ++ cat /tmp/tmp.NmAK4zgPl3 ++ rm /tmp/tmp.Oj05WDqbww /tmp/tmp.NmAK4zgPl3 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gDWwbExnU3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ev9Jcl4APx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gDWwbExnU3 ++ cat /tmp/tmp.ev9Jcl4APx ++ rm /tmp/tmp.gDWwbExnU3 /tmp/tmp.ev9Jcl4APx ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 05:04:33.332482 32242 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968671897142000&timeoutSeconds=570&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data smart-update 100500 no-proxy -P3306 + local cluster=smart-update + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.v0pCaM7FN2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RL0JiWGy5N +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.v0pCaM7FN2 +++ cat /tmp/tmp.RL0JiWGy5N +++ rm /tmp/tmp.v0pCaM7FN2 /tmp/tmp.RL0JiWGy5N +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.T09VeA3yBP +++ mktemp ++ local LAST_ERR=/tmp/tmp.x4gAgTAjhc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.T09VeA3yBP ++ cat /tmp/tmp.x4gAgTAjhc ++ rm /tmp/tmp.T09VeA3yBP /tmp/tmp.x4gAgTAjhc ++ return 0 + root_pass=root_password ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.FYTBY9Clgk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GeDU7tHdbH +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.FYTBY9Clgk +++ cat /tmp/tmp.GeDU7tHdbH +++ rm /tmp/tmp.FYTBY9Clgk /tmp/tmp.GeDU7tHdbH +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + proxy=smart-update-haproxy + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h smart-update-pxc-0.smart-update-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FRAAxQztyW +++ mktemp ++ local LAST_ERR=/tmp/tmp.AskY29yhcP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FRAAxQztyW ++ cat /tmp/tmp.AskY29yhcP ++ rm /tmp/tmp.FRAAxQztyW /tmp/tmp.AskY29yhcP ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 05:04:44.392401 1293 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968682723411000&timeoutSeconds=337&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data smart-update select-1 no-proxy -P3306 + local cluster=smart-update + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.AJyjGVJrW7 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Ct5RwwWAwv +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.AJyjGVJrW7 +++ cat /tmp/tmp.Ct5RwwWAwv +++ rm /tmp/tmp.AJyjGVJrW7 /tmp/tmp.Ct5RwwWAwv +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.O0s8Fcf3Zk +++ mktemp ++ local LAST_ERR=/tmp/tmp.n0iOc1JT34 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.O0s8Fcf3Zk ++ cat /tmp/tmp.n0iOc1JT34 ++ rm /tmp/tmp.O0s8Fcf3Zk /tmp/tmp.n0iOc1JT34 ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NKjK57WZHP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1IsQl64pg4 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.NKjK57WZHP +++ cat /tmp/tmp.1IsQl64pg4 +++ rm /tmp/tmp.NKjK57WZHP /tmp/tmp.1IsQl64pg4 +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DLX0vF8UGE +++ mktemp ++ local LAST_ERR=/tmp/tmp.6OROoHzcVf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DLX0vF8UGE ++ cat /tmp/tmp.6OROoHzcVf ++ rm /tmp/tmp.DLX0vF8UGE /tmp/tmp.6OROoHzcVf ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 05:05:26.863362 5716 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968724726918000&timeoutSeconds=587&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uQHTjC3BGt +++ mktemp ++ local LAST_ERR=/tmp/tmp.KxO56izd0Y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uQHTjC3BGt ++ cat /tmp/tmp.KxO56izd0Y ++ rm /tmp/tmp.uQHTjC3BGt /tmp/tmp.KxO56izd0Y ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 05:05:36.606334 7124 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968734933146000&timeoutSeconds=404&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AOiFNKQ9fO +++ mktemp ++ local LAST_ERR=/tmp/tmp.XfZF0UGMzU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AOiFNKQ9fO ++ cat /tmp/tmp.XfZF0UGMzU ++ rm /tmp/tmp.AOiFNKQ9fO /tmp/tmp.XfZF0UGMzU ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6666d5df5c-7gzlp ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 05:05:44.143056 8241 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968742560086000&timeoutSeconds=304&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + kubectl exec smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + grep -E -o 'early-plugin-load=keyring_\w+.so' + return 1 ++ jq -r '.versions[].matrix.pxc[].imagePath' +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++ head -n1 ++ tail -n2 ++ sort -V ++++ grep -c percona-xtradb-cluster-operator ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ grep :8.0 +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ head -1 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.utITUxsyNp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cX5oV51aVi +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.utITUxsyNp +++ cat /tmp/tmp.cX5oV51aVi +++ rm /tmp/tmp.utITUxsyNp /tmp/tmp.cX5oV51aVi +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-7c6fd7c9f9-rzqrt -n pxc-operator -- curl -s http://version-service.smart-update2-30577.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.IQ4hM1gqZy +++ mktemp ++ local LAST_ERR=/tmp/tmp.F7WMuMAxo3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-7c6fd7c9f9-rzqrt -n pxc-operator -- curl -s http://version-service.smart-update2-30577.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IQ4hM1gqZy ++ cat /tmp/tmp.F7WMuMAxo3 Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.IQ4hM1gqZy /tmp/tmp.F7WMuMAxo3 ++ return 0 + vs_image=percona/percona-xtradb-cluster:8.0.41-32.1 ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BH8z3lBp7M ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MDdS50sLl3 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.BH8z3lBp7M +++ cat /tmp/tmp.MDdS50sLl3 +++ rm /tmp/tmp.BH8z3lBp7M /tmp/tmp.MDdS50sLl3 +++ return 0 ++ client_pod=pxc-client-6666d5df5c-7gzlp ++ wait_pod pxc-client-6666d5df5c-7gzlp ++ local pod=pxc-client-6666d5df5c-7gzlp ++ local max_retry=480 ++ local ns= +++ echo pxc-client-6666d5df5c-7gzlp +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ grep -E '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 05:05:55.118643 9588 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968753668390000&timeoutSeconds=488&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.41-32.1","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.fofuBaHXQg ++ mktemp + local LAST_ERR=/tmp/tmp.Ocn7hokZQe + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.41-32.1","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fofuBaHXQg perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.Ocn7hokZQe + rm /tmp/tmp.fofuBaHXQg /tmp/tmp.Ocn7hokZQe + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.41-32.1 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.41-32.1 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eNiZUHRiUb +++ mktemp ++ local LAST_ERR=/tmp/tmp.dfvqpd24Zo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eNiZUHRiUb ++ cat /tmp/tmp.dfvqpd24Zo ++ rm /tmp/tmp.eNiZUHRiUb /tmp/tmp.dfvqpd24Zo ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wxDyOw8ccO +++ mktemp ++ local LAST_ERR=/tmp/tmp.EylBogtirL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wxDyOw8ccO ++ cat /tmp/tmp.EylBogtirL ++ rm /tmp/tmp.wxDyOw8ccO /tmp/tmp.EylBogtirL ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.gs2qwF4Bhj ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.R17hh5Mw9U +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.gs2qwF4Bhj +++++ cat /tmp/tmp.R17hh5Mw9U +++++ rm /tmp/tmp.gs2qwF4Bhj /tmp/tmp.R17hh5Mw9U +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sUr35wIS3y +++ mktemp ++ local LAST_ERR=/tmp/tmp.B0cZAgyqAj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sUr35wIS3y ++ cat /tmp/tmp.B0cZAgyqAj ++ rm /tmp/tmp.sUr35wIS3y /tmp/tmp.B0cZAgyqAj ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fW3aspdM5g +++ mktemp ++ local LAST_ERR=/tmp/tmp.oORkFTMLDG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fW3aspdM5g ++ cat /tmp/tmp.oORkFTMLDG ++ rm /tmp/tmp.fW3aspdM5g /tmp/tmp.oORkFTMLDG ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 05:07:09.340357 18149 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968827717908000&timeoutSeconds=313&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BF5qaPIsxp +++ mktemp ++ local LAST_ERR=/tmp/tmp.ebnIktDSxx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BF5qaPIsxp ++ cat /tmp/tmp.ebnIktDSxx ++ rm /tmp/tmp.BF5qaPIsxp /tmp/tmp.ebnIktDSxx ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 05:07:20.423420 19818 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968838888057000&timeoutSeconds=463&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pXuhlb7VAY +++ mktemp ++ local LAST_ERR=/tmp/tmp.JqgLzmaWR1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pXuhlb7VAY ++ cat /tmp/tmp.JqgLzmaWR1 ++ rm /tmp/tmp.pXuhlb7VAY /tmp/tmp.JqgLzmaWR1 ++ return 0 + client_pod=pxc-client-6666d5df5c-7gzlp + wait_pod pxc-client-6666d5df5c-7gzlp + local pod=pxc-client-6666d5df5c-7gzlp + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-7gzlp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-7gzlp condition met E0702 05:07:31.613042 21364 reflector.go:227] "Failed to watch" err="Get \"https://35.232.252.217/api/v1/namespaces/smart-update2-30577/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-7gzlp&resourceVersion=1782968849107912000&timeoutSeconds=471&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-7gzlp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.rCF6hRfwxh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.rCF6hRfwxh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.rCF6hRfwxh/select-1.sql + kubectl_bin delete -f /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.RdREhfHEEd ++ mktemp + local LAST_ERR=/tmp/tmp.QQXkFF9j4t + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.rCF6hRfwxh/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RdREhfHEEd perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-30577 namespace + cat /tmp/tmp.QQXkFF9j4t + rm /tmp/tmp.RdREhfHEEd /tmp/tmp.QQXkFF9j4t + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.b3MpkoqLf2 ++ mktemp + local LAST_ERR=/tmp/tmp.MKtOtJKeQY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.b3MpkoqLf2 persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-30577 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-30577 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-30577 namespace + cat /tmp/tmp.MKtOtJKeQY + rm /tmp/tmp.b3MpkoqLf2 /tmp/tmp.MKtOtJKeQY + return 0 + desc cleanup + set +o xtrace ----------------------------------------------------------------------------------- cleanup ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.V1kZzc7Df0 ++ mktemp + local LAST_ERR=/tmp/tmp.Vs9cwFhhdV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.V1kZzc7Df0 deployment.apps "version-service" deleted from smart-update2-30577 namespace service "version-service" deleted from smart-update2-30577 namespace + cat /tmp/tmp.Vs9cwFhhdV + rm /tmp/tmp.V1kZzc7Df0 /tmp/tmp.Vs9cwFhhdV + return 0 + destroy smart-update2-30577 + local namespace=smart-update2-30577 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'the object has been modified' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator + grep -v 'get backup status: Job.batch' +++ grep -c percona-xtradb-cluster-operator + grep -v level=info + sort -u + tee /tmp/tmp.rCF6hRfwxh/operator.log ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vEdlQ12aEN +++ mktemp ++ local LAST_ERR=/tmp/tmp.dTHrqtUToy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vEdlQ12aEN ++ cat /tmp/tmp.dTHrqtUToy ++ rm /tmp/tmp.vEdlQ12aEN /tmp/tmp.dTHrqtUToy ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-7c6fd7c9f9-rzqrt ++ mktemp + local LAST_OUT=/tmp/tmp.7frU49OfN0 ++ mktemp + local LAST_ERR=/tmp/tmp.am3Luvmqgg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-7c6fd7c9f9-rzqrt + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7frU49OfN0 + cat /tmp/tmp.am3Luvmqgg + rm /tmp/tmp.7frU49OfN0 /tmp/tmp.am3Luvmqgg + return 0 2026-07-02T04:37:31.071Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-07-02T04:37:31.071Z INFO setup Manager starting up {"gitCommit": "70c04858efdd60ae80fd1cf4dcb3edc388dbd027", "gitBranch": "PR-2538-70c04858", "buildTime": "2026-07-02T01:14:54Z", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} 2026-07-02T04:37:31.071Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.33.12-gke.1059000"} 2026-07-02T04:37:31.074Z INFO setup Registering Components. 2026-07-02T04:37:31.167Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-07-02T04:37:31.167Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-07-02T04:37:31.167Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-07-02T04:37:31.167Z INFO controller-runtime.metrics Starting metrics server 2026-07-02T04:37:31.167Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-07-02T04:37:31.167Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-07-02T04:37:31.167Z INFO controller-runtime.webhook Starting webhook server 2026-07-02T04:37:31.167Z INFO setup Starting the Cmd. 2026-07-02T04:37:31.167Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-07-02T04:37:31.367Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-07-02T04:37:53.689Z DEBUG events percona-xtradb-cluster-operator-7c6fd7c9f9-rzqrt_3c6d1946-f266-46a9-8470-602b192db1c5 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"d3840e46-c4bd-455d-b86d-335d81d650cd","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1782967073683343009"}, "reason": "LeaderElection"} 2026-07-02T04:37:53.689Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-07-02T04:37:53.689Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-07-02T04:37:53.689Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-07-02T04:37:53.689Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-07-02T04:37:53.689Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-07-02T04:37:53.789Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-07-02T04:37:53.790Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-07-02T04:37:53.890Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-07-02T04:37:53.890Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-07-02T04:37:53.890Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-07-02T04:37:53.890Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-07-02T04:37:58.843Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"minimal-cluster","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "minimal-cluster", "reconcileID": "ac7ed156-a032-4103-afb5-89a096406112", "error": "PerconaXtraDBCluster.pxc.percona.com \"minimal-cluster\" not found"} 2026-07-02T04:38:53.794Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "d546a573-5c87-4d40-a892-2d3162fabd80", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:38:53.810Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "d546a573-5c87-4d40-a892-2d3162fabd80", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:38:54.369Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "d546a573-5c87-4d40-a892-2d3162fabd80", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:38:54.465Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "fc17ff8a-36e5-4c30-b6c0-eb333d8bc15d", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:38:54.508Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "fc17ff8a-36e5-4c30-b6c0-eb333d8bc15d", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:38:54.582Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "fc17ff8a-36e5-4c30-b6c0-eb333d8bc15d", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:38:54.624Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "fc17ff8a-36e5-4c30-b6c0-eb333d8bc15d", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:38:54.679Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "fc17ff8a-36e5-4c30-b6c0-eb333d8bc15d", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:38:54.853Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "fc17ff8a-36e5-4c30-b6c0-eb333d8bc15d", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:38:55.783Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "7b1b9cf2-31a2-4af5-b66c-ca928a83491f", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:38:55.807Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "7b1b9cf2-31a2-4af5-b66c-ca928a83491f", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:40:17.153Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "c92dee55-be36-4ec3-9a69-ba53728c9dea", "user": "operator"} 2026-07-02T04:40:17.197Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "c92dee55-be36-4ec3-9a69-ba53728c9dea", "user": "monitor"} 2026-07-02T04:40:17.255Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "c92dee55-be36-4ec3-9a69-ba53728c9dea"} 2026-07-02T04:40:17.299Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "c92dee55-be36-4ec3-9a69-ba53728c9dea"} 2026-07-02T04:40:17.346Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "c92dee55-be36-4ec3-9a69-ba53728c9dea", "user": "xtrabackup"} 2026-07-02T04:40:17.391Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "c92dee55-be36-4ec3-9a69-ba53728c9dea"} 2026-07-02T04:40:17.421Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "c92dee55-be36-4ec3-9a69-ba53728c9dea", "user": "replication"} 2026-07-02T04:43:03.278Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "63389a51-4f06-40e4-8f0b-a3c63b17ef0f", "user": "root"} 2026-07-02T04:43:04.384Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "63389a51-4f06-40e4-8f0b-a3c63b17ef0f", "new version": "8.0.41-32.1"} 2026-07-02T04:46:34.190Z INFO add new job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "a62a9957-d0dc-46ab-8d65-2009be8f8fcd", "name": "ensure-version/smart-update2-30577/smart-update", "schedule": "* * * * *"} 2026-07-02T04:47:34.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "a62a9957-d0dc-46ab-8d65-2009be8f8fcd", "endpoint": "http://version-service.smart-update2-30577.svc.cluster.local:11000"} 2026-07-02T04:47:34.089Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "a62a9957-d0dc-46ab-8d65-2009be8f8fcd", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=10d28753-34d9-40ac-85e7-dffec13a5ea2&databaseVersion=8.0.41-32.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=10d28753-34d9-40ac-85e7-dffec13a5ea2&databaseVersion=8.0.41-32.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:48:34.001Z INFO cluster is not ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "a62a9957-d0dc-46ab-8d65-2009be8f8fcd"} 2026-07-02T04:49:10.237Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "df8cb5c6-05cc-484f-8425-763037c71d50", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2026-07-02T04:49:33.751Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "7872e62a-868e-446e-a7d4-4af6a3908c0d", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:49:33.767Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "7872e62a-868e-446e-a7d4-4af6a3908c0d", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:49:34.321Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "7872e62a-868e-446e-a7d4-4af6a3908c0d", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:49:34.321Z INFO cluster is not ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "a62a9957-d0dc-46ab-8d65-2009be8f8fcd"} 2026-07-02T04:49:34.417Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "bc0d2b53-8aa2-44e2-838a-2fff6c369f1c", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:49:34.457Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "bc0d2b53-8aa2-44e2-838a-2fff6c369f1c", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:49:34.504Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "bc0d2b53-8aa2-44e2-838a-2fff6c369f1c", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:49:34.558Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "bc0d2b53-8aa2-44e2-838a-2fff6c369f1c", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:49:34.599Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "bc0d2b53-8aa2-44e2-838a-2fff6c369f1c", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:49:34.835Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "bc0d2b53-8aa2-44e2-838a-2fff6c369f1c", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:49:35.723Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "08b3bf1e-0158-44f9-ae14-5ea55e9aee5c", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:49:35.743Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "08b3bf1e-0158-44f9-ae14-5ea55e9aee5c", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:50:59.143Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "ffef16d7-5548-42b9-a0ed-bf97b7b53cf4", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.237.77:3306: connect: connection refused"} 2026-07-02T04:53:33.895Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "74ac5bf6-23fa-458e-af1e-80d67da4c80b", "new version": "8.0.41-32.1"} 2026-07-02T04:56:50.480Z INFO add new job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "8398ac09-ba43-4b35-9919-29dc993a33e7", "name": "ensure-version/smart-update2-30577/smart-update", "schedule": "* * * * *"} 2026-07-02T04:57:50.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "8398ac09-ba43-4b35-9919-29dc993a33e7", "endpoint": "http://version-service.smart-update2-30577.svc.cluster.local:11000"} 2026-07-02T04:57:50.447Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "8398ac09-ba43-4b35-9919-29dc993a33e7", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=a03e910c-9ad4-4f49-bf62-c51ef05041ed&databaseVersion=8.0.41-32.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=a03e910c-9ad4-4f49-bf62-c51ef05041ed&databaseVersion=8.0.41-32.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:58:51.207Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "8398ac09-ba43-4b35-9919-29dc993a33e7", "endpoint": "http://version-service.smart-update2-30577.svc.cluster.local:11000"} 2026-07-02T04:58:51.240Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "8398ac09-ba43-4b35-9919-29dc993a33e7", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=a03e910c-9ad4-4f49-bf62-c51ef05041ed&databaseVersion=8.0.41-32.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=a03e910c-9ad4-4f49-bf62-c51ef05041ed&databaseVersion=8.0.41-32.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:59:31.527Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "1efef3f5-42a7-4383-881a-2e835ed0fd72", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2026-07-02T04:59:49.310Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "6d7c1f58-d37f-4d61-9092-cfc53f8fc64f", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:59:49.324Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "6d7c1f58-d37f-4d61-9092-cfc53f8fc64f", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:59:49.873Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "6d7c1f58-d37f-4d61-9092-cfc53f8fc64f", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:59:49.968Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "b50053c1-c402-47f0-a4db-0803d4c8e00d", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:59:50.012Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "b50053c1-c402-47f0-a4db-0803d4c8e00d", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:59:50.070Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "b50053c1-c402-47f0-a4db-0803d4c8e00d", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:59:50.126Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "b50053c1-c402-47f0-a4db-0803d4c8e00d", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:59:50.191Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "b50053c1-c402-47f0-a4db-0803d4c8e00d", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:59:50.309Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "b50053c1-c402-47f0-a4db-0803d4c8e00d", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:59:51.085Z INFO cluster is not ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "8398ac09-ba43-4b35-9919-29dc993a33e7"} 2026-07-02T04:59:51.095Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "700853d7-f79f-47e8-85e0-ae27b95a7438", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:59:51.111Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "700853d7-f79f-47e8-85e0-ae27b95a7438", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T05:03:53.371Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "c9302c43-39e5-49c6-b987-769274758ba7", "new version": "8.0.41-32.1"} 2026-07-02T05:06:01.328Z INFO add new job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "89634cb9-44e2-47f0-ae59-399fbde879e0", "name": "ensure-version/smart-update2-30577/smart-update", "schedule": "* * * * *"} 2026-07-02T05:07:01.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "89634cb9-44e2-47f0-ae59-399fbde879e0", "endpoint": "http://version-service.smart-update2-30577.svc.cluster.local:11000"} 2026-07-02T05:07:01.071Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "89634cb9-44e2-47f0-ae59-399fbde879e0", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.41-32.1?clusterWideEnabled=true&customResourceUid=24f828ee-b87a-4989-b75d-5cab9553fd51&databaseVersion=8.0.41-32.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.41-32.1?clusterWideEnabled=true&customResourceUid=24f828ee-b87a-4989-b75d-5cab9553fd51&databaseVersion=8.0.41-32.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T05:08:01.001Z INFO cluster is not ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "89634cb9-44e2-47f0-ae59-399fbde879e0"} 2026-07-02T05:08:19.282Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-30577"}, "namespace": "smart-update2-30577", "name": "smart-update", "reconcileID": "d0ff8a3f-ee21-479b-9df2-ee48ae91c410", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1 github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1 github.com/robfig/cron/v3.(*Cron).startJob.func1 github.com/robfig/cron/v3.FuncJob.Run /go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136 /go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:272 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:509 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:215 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 + local backup_names ++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + backup_names= + [[ -n '' ]] + local pxc_names ++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + pxc_names= + [[ -n '' ]] + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.yBZpQA75Ww ++ mktemp + local LAST_ERR=/tmp/tmp.bVQa4ug8FG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yBZpQA75Ww validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.bVQa4ug8FG + rm /tmp/tmp.yBZpQA75Ww /tmp/tmp.bVQa4ug8FG + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml namespace "cert-manager" deleted customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted serviceaccount "cert-manager-cainjector" deleted from cert-manager namespace serviceaccount "cert-manager" deleted from cert-manager namespace serviceaccount "cert-manager-webhook" deleted from cert-manager namespace clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace role.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace service "cert-manager-cainjector" deleted from cert-manager namespace service "cert-manager" deleted from cert-manager namespace service "cert-manager-webhook" deleted from cert-manager namespace deployment.apps "cert-manager-cainjector" deleted from cert-manager namespace deployment.apps "cert-manager" deleted from cert-manager namespace deployment.apps "cert-manager-webhook" deleted from cert-manager namespace mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted + '[' '!' -z '' ']' + '[' -n pxc-operator ']' ++ date +%s + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator + kubectl_bin delete --grace-period=0 --force=true namespace smart-update2-30577 ++ mktemp + local deadline=1782969068 + local LAST_OUT=/tmp/tmp.ZafmIFCWrO ++ mktemp ++ kubectl_bin get namespace smart-update2-30577 --ignore-not-found -o name + local LAST_OUT=/tmp/tmp.YQSkhdSl5d ++ mktemp + local LAST_ERR=/tmp/tmp.708Hwxzk1f + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp + local LAST_ERR=/tmp/tmp.DPQZPPL8Lv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace smart-update2-30577 + [[ -z namespace/smart-update2-30577 ]] ++ date +%s + [[ 1782968950 -ge 1782969068 ]] + sleep 5 ++ kubectl_bin get namespace smart-update2-30577 --ignore-not-found -o name + [[ -z namespace/smart-update2-30577 ]] ++ date +%s + [[ 1782968957 -ge 1782969068 ]] + sleep 5 ++ kubectl_bin get namespace smart-update2-30577 --ignore-not-found -o name + [[ -z namespace/smart-update2-30577 ]] ++ date +%s + [[ 1782968964 -ge 1782969068 ]] + sleep 5 ++ kubectl_bin get namespace smart-update2-30577 --ignore-not-found -o name + [[ -z namespace/smart-update2-30577 ]] ++ date +%s + [[ 1782968971 -ge 1782969068 ]] + sleep 5 ++ kubectl_bin get namespace smart-update2-30577 --ignore-not-found -o name + [[ -z namespace/smart-update2-30577 ]] ++ date +%s + [[ 1782968978 -ge 1782969068 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YQSkhdSl5d namespace "smart-update2-30577" force deleted + cat /tmp/tmp.DPQZPPL8Lv Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.YQSkhdSl5d /tmp/tmp.DPQZPPL8Lv + return 0 ++ kubectl_bin get namespace smart-update2-30577 --ignore-not-found -o name + [[ -z '' ]] ++ date +%s + deadline=1782969105 ++ awk '$2 != ""' ++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' + [[ -z '' ]] + rm -rf /tmp/tmp.rCF6hRfwxh + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------