Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/logs/smart-update2-8-4.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + API=pxc.percona.com/v9-9-9 + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + CLUSTER=smart-update + CLUSTER_SIZE=3 + PROXY_SIZE=2 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + PXC_VER=8.4 + TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + VS_URL=http://version-service + VS_PORT=11000 + VS_ENDPOINT=http://version-service:11000 + [[ 8.4 == \8\.\4 ]] + VS_UPDATE_STRATEGY=latest + main + create_infra smart-update2-6497 + local ns=smart-update2-6497 + '[' -n pxc-operator ']' + read -r pns name + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl api-resources ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.BvI6zl8MQJ + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp ++ mktemp + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.z4Igtb2NHt + local LAST_ERR=/tmp/tmp.2jL50w0Wzn + local exit_status=0 ++ seq 0 2 ++ mktemp + for i in '$(seq 0 2)' + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.3fX9xKRkYz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BvI6zl8MQJ + cat /tmp/tmp.2jL50w0Wzn + rm /tmp/tmp.BvI6zl8MQJ /tmp/tmp.2jL50w0Wzn + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.z4Igtb2NHt namespace "pxc-operator" deleted + cat /tmp/tmp.3fX9xKRkYz + rm /tmp/tmp.z4Igtb2NHt /tmp/tmp.3fX9xKRkYz + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.AMP3TFNk16 ++ mktemp + local LAST_ERR=/tmp/tmp.yJsMjSNHHc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AMP3TFNk16 namespace/pxc-operator created + cat /tmp/tmp.yJsMjSNHHc + rm /tmp/tmp.AMP3TFNk16 /tmp/tmp.yJsMjSNHHc + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.wPKfbqimbH +++ mktemp ++ local LAST_ERR=/tmp/tmp.pIHfEKlleK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wPKfbqimbH ++ cat /tmp/tmp.pIHfEKlleK ++ rm /tmp/tmp.wPKfbqimbH /tmp/tmp.pIHfEKlleK ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster8 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.bMsHi6xb0a ++ mktemp + local LAST_ERR=/tmp/tmp.0ziVfx7kxX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster8 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bMsHi6xb0a Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster8" modified. + cat /tmp/tmp.0ziVfx7kxX + rm /tmp/tmp.bMsHi6xb0a /tmp/tmp.0ziVfx7kxX + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.Jt3Meq0F9B ++ mktemp + local LAST_ERR=/tmp/tmp.ARX2gv8eZC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Jt3Meq0F9B customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.ARX2gv8eZC + rm /tmp/tmp.Jt3Meq0F9B /tmp/tmp.ARX2gv8eZC + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.0rQ1ivgVU0 ++ mktemp + local LAST_ERR=/tmp/tmp.B71vi6bFIQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0rQ1ivgVU0 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.B71vi6bFIQ E0702 04:18:00.889142 29080 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusters.pxc.percona.com&resourceVersion=1782965879787149000&timeoutSeconds=470&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.0rQ1ivgVU0 /tmp/tmp.B71vi6bFIQ + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.AsRxG3rchT ++ mktemp + local LAST_ERR=/tmp/tmp.IRcqVDtxRo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AsRxG3rchT customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.IRcqVDtxRo E0702 04:18:02.796541 29321 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterbackups.pxc.percona.com&resourceVersion=1782965879787149000&timeoutSeconds=373&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.AsRxG3rchT /tmp/tmp.IRcqVDtxRo + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.IuqhSRfXke ++ mktemp + local LAST_ERR=/tmp/tmp.PLiPcNJey6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IuqhSRfXke customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.PLiPcNJey6 E0702 04:18:04.764759 29625 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterrestores.pxc.percona.com&resourceVersion=1782965879787149000&timeoutSeconds=440&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.IuqhSRfXke /tmp/tmp.PLiPcNJey6 + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/cw-rbac.yaml + kubectl_bin apply -f - ++ mktemp + sed -e 's^namespace: .*^namespace: pxc-operator^' + local LAST_OUT=/tmp/tmp.7xD3k6QfKK ++ mktemp + local LAST_ERR=/tmp/tmp.e5uUnd90rd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7xD3k6QfKK clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.e5uUnd90rd + rm /tmp/tmp.7xD3k6QfKK /tmp/tmp.e5uUnd90rd + return 0 + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/cw-operator.yaml + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ardCtjmxkt ++ mktemp + local LAST_ERR=/tmp/tmp.5VfZIQNmvd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ardCtjmxkt deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.5VfZIQNmvd + rm /tmp/tmp.ardCtjmxkt /tmp/tmp.5VfZIQNmvd + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.F0B8JWZ6Sa ++ mktemp + local LAST_ERR=/tmp/tmp.8d6VwXUTA7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.F0B8JWZ6Sa pod/percona-xtradb-cluster-operator-59848dbdcc-pwl2x condition met + cat /tmp/tmp.8d6VwXUTA7 E0702 04:18:21.840199 31824 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-59848dbdcc-pwl2x&resourceVersion=1782965901479451000&timeoutSeconds=381&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.F0B8JWZ6Sa /tmp/tmp.8d6VwXUTA7 + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.wE5uMRxVr3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.csyHlDGES7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wE5uMRxVr3 ++ cat /tmp/tmp.csyHlDGES7 ++ rm /tmp/tmp.wE5uMRxVr3 /tmp/tmp.csyHlDGES7 ++ return 0 + wait_pod percona-xtradb-cluster-operator-59848dbdcc-pwl2x 480 pxc-operator + local pod=percona-xtradb-cluster-operator-59848dbdcc-pwl2x + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-59848dbdcc-pwl2x ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-59848dbdcc-pwl2x condition met E0702 04:18:26.070004 32418 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-59848dbdcc-pwl2x&resourceVersion=1782965904389089000&timeoutSeconds=457&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-59848dbdcc-pwl2x to become Ready.Ok + sleep 3 + create_namespace smart-update2-6497 + local namespace=smart-update2-6497 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces smart-update2-6497' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces smart-update2-6497 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace smart-update2-6497 + xargs kubectl delete ns + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + local LAST_OUT=/tmp/tmp.4ul7ge7tPl + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.yhaNk0DwZv + local exit_status=0 + local LAST_OUT=/tmp/tmp.iQ2otWSfd7 ++ seq 0 2 ++ mktemp + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-6497 + local LAST_ERR=/tmp/tmp.tVmxyztpF2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-6497 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iQ2otWSfd7 + cat /tmp/tmp.tVmxyztpF2 + rm /tmp/tmp.iQ2otWSfd7 /tmp/tmp.tVmxyztpF2 + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-6497 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.4ul7ge7tPl + cat /tmp/tmp.yhaNk0DwZv Error from server (NotFound): namespaces "smart-update2-6497" not found + rm /tmp/tmp.4ul7ge7tPl /tmp/tmp.yhaNk0DwZv + return 1 + : + wait_for_delete namespace/smart-update2-6497 + local res=namespace/smart-update2-6497 + echo -n 'waiting for namespace/smart-update2-6497 to be deleted' waiting for namespace/smart-update2-6497 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "smart-update2-6497" not found + desc 'create namespace smart-update2-6497' + set +o xtrace ----------------------------------------------------------------------------------- create namespace smart-update2-6497 ----------------------------------------------------------------------------------- + kubectl_bin create namespace smart-update2-6497 ++ mktemp + local LAST_OUT=/tmp/tmp.o8grjzgYKk ++ mktemp + local LAST_ERR=/tmp/tmp.ZTp0CY8fQO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace smart-update2-6497 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.o8grjzgYKk namespace/smart-update2-6497 created + cat /tmp/tmp.ZTp0CY8fQO + rm /tmp/tmp.o8grjzgYKk /tmp/tmp.ZTp0CY8fQO + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.NS7IVCokyi +++ mktemp ++ local LAST_ERR=/tmp/tmp.nKUnH75anW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NS7IVCokyi ++ cat /tmp/tmp.nKUnH75anW ++ rm /tmp/tmp.NS7IVCokyi /tmp/tmp.nKUnH75anW ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster8 --namespace=smart-update2-6497 ++ mktemp + local LAST_OUT=/tmp/tmp.s8WNTEHGjK ++ mktemp + local LAST_ERR=/tmp/tmp.6voDyzkltg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster8 --namespace=smart-update2-6497 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.s8WNTEHGjK Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster8" modified. + cat /tmp/tmp.6voDyzkltg + rm /tmp/tmp.s8WNTEHGjK /tmp/tmp.6voDyzkltg + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.g1KTHtt6Wq ++ mktemp + local LAST_ERR=/tmp/tmp.g0Vk7FbQou + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.g1KTHtt6Wq secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.g0Vk7FbQou + rm /tmp/tmp.g1KTHtt6Wq /tmp/tmp.g0Vk7FbQou + return 0 + deploy_version_service + desc 'install version service' + set +o xtrace ----------------------------------------------------------------------------------- install version service ----------------------------------------------------------------------------------- + kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json ++ mktemp + local LAST_OUT=/tmp/tmp.sC1QhABRk2 ++ mktemp + local LAST_ERR=/tmp/tmp.2Ln5nP7pDk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sC1QhABRk2 configmap/versions created + cat /tmp/tmp.2Ln5nP7pDk + rm /tmp/tmp.sC1QhABRk2 /tmp/tmp.2Ln5nP7pDk + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xgKgcsHnml ++ mktemp + local LAST_ERR=/tmp/tmp.VdZ6cbI265 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xgKgcsHnml deployment.apps/version-service created service/version-service created + cat /tmp/tmp.VdZ6cbI265 + rm /tmp/tmp.xgKgcsHnml /tmp/tmp.VdZ6cbI265 + return 0 + sleep 10 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.sabXzE8u2I ++ mktemp + local LAST_ERR=/tmp/tmp.c300IXUZWl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sabXzE8u2I namespace/cert-manager created + cat /tmp/tmp.c300IXUZWl + rm /tmp/tmp.sabXzE8u2I /tmp/tmp.c300IXUZWl + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.xMUEH41QEc ++ mktemp + local LAST_ERR=/tmp/tmp.Toz1rjY3N4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xMUEH41QEc namespace/cert-manager labeled + cat /tmp/tmp.Toz1rjY3N4 + rm /tmp/tmp.xMUEH41QEc /tmp/tmp.Toz1rjY3N4 + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.VM9RnW2lmN ++ mktemp + local LAST_ERR=/tmp/tmp.vmE2da7mMf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VM9RnW2lmN namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io created serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view created clusterrole.rbac.authorization.k8s.io/cert-manager-view created clusterrole.rbac.authorization.k8s.io/cert-manager-edit created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created role.rbac.authorization.k8s.io/cert-manager:leaderelection created role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created + cat /tmp/tmp.vmE2da7mMf Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.VM9RnW2lmN /tmp/tmp.vmE2da7mMf + return 0 + '[' '' == 4.10 ']' + kubectl_bin -n cert-manager rollout status deployment/cert-manager --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.Z8Es5QNjT6 ++ mktemp + local LAST_ERR=/tmp/tmp.d9i9h4Gbqi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n cert-manager rollout status deployment/cert-manager --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Z8Es5QNjT6 Waiting for deployment "cert-manager" rollout to finish: 0 of 1 updated replicas are available... deployment "cert-manager" successfully rolled out + cat /tmp/tmp.d9i9h4Gbqi + rm /tmp/tmp.Z8Es5QNjT6 /tmp/tmp.d9i9h4Gbqi + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.RJ6ew44T0Q ++ mktemp + local LAST_ERR=/tmp/tmp.DLFlX2hoQO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RJ6ew44T0Q deployment "cert-manager-cainjector" successfully rolled out + cat /tmp/tmp.DLFlX2hoQO + rm /tmp/tmp.RJ6ew44T0Q /tmp/tmp.DLFlX2hoQO + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.OLTQO2qz8j ++ mktemp + local LAST_ERR=/tmp/tmp.tikmVN7spE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OLTQO2qz8j Waiting for deployment "cert-manager-webhook" rollout to finish: 0 of 1 updated replicas are available... deployment "cert-manager-webhook" successfully rolled out + cat /tmp/tmp.tikmVN7spE + rm /tmp/tmp.OLTQO2qz8j /tmp/tmp.tikmVN7spE + return 0 + log 'waiting for cert-manager webhook caBundle to be injected' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:19:33+0000]' waiting for cert-manager webhook caBundle to be injected [2026-07-02T04:19:33+0000] waiting for cert-manager webhook caBundle to be injected ++ date +%s + local deadline=1782966093 + kubectl_bin get validatingwebhookconfigurations cert-manager-webhook -o 'jsonpath={.webhooks[0].clientConfig.caBundle}' + grep -q . + log 'probing cert-manager webhook' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:19:34+0000]' probing cert-manager webhook [2026-07-02T04:19:34+0000] probing cert-manager webhook + local probe_file ++ mktemp + probe_file=/tmp/tmp.iOCjjsHLBo + cat + local retries=0 + kubectl_bin apply -f /tmp/tmp.iOCjjsHLBo --dry-run=server + rm -f /tmp/tmp.iOCjjsHLBo ++ jq -r '.versions[].matrix.pxc[].imagePath' +++ get_operator_pod ++ grep :8.4 +++ local label_prefix=app.kubernetes.io/ ++ tail -n3 ++ head -n1 ++ sort -V ++++ grep -c percona-xtradb-cluster-operator ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ head -1 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.89Wk1zbh3v ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YUN5trIAp5 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.89Wk1zbh3v +++ cat /tmp/tmp.YUN5trIAp5 +++ rm /tmp/tmp.89Wk1zbh3v /tmp/tmp.YUN5trIAp5 +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-59848dbdcc-pwl2x -n pxc-operator -- curl -s http://version-service.smart-update2-6497.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.qzxKhUiksU +++ mktemp ++ local LAST_ERR=/tmp/tmp.g2P7sLhLiR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-59848dbdcc-pwl2x -n pxc-operator -- curl -s http://version-service.smart-update2-6497.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qzxKhUiksU ++ cat /tmp/tmp.g2P7sLhLiR Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.qzxKhUiksU /tmp/tmp.g2P7sLhLiR ++ return 0 + IMAGE_PXC=percona/percona-xtradb-cluster:8.4.5-5.1 + kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' ++ mktemp + local LAST_OUT=/tmp/tmp.mqQSGDebsG ++ mktemp + local LAST_ERR=/tmp/tmp.B02wNBvlLK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mqQSGDebsG customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched + cat /tmp/tmp.B02wNBvlLK + rm /tmp/tmp.mqQSGDebsG /tmp/tmp.B02wNBvlLK + return 0 + kubectl_bin -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-6497.svc.cluster.local:11000 ++ mktemp + local LAST_OUT=/tmp/tmp.UvScTeTYmB ++ mktemp + local LAST_ERR=/tmp/tmp.K16hLV46mK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-6497.svc.cluster.local:11000 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UvScTeTYmB deployment.apps/percona-xtradb-cluster-operator env updated + cat /tmp/tmp.K16hLV46mK + rm /tmp/tmp.UvScTeTYmB /tmp/tmp.K16hLV46mK + return 0 + desc 'Starting telemetry testing' + set +o xtrace ----------------------------------------------------------------------------------- Starting telemetry testing ----------------------------------------------------------------------------------- + /usr/bin/sed s/version-service/version-service-cr/g /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/vs.yml + kubectl_bin apply -f - ++ yq 'select(.kind == "Deployment").spec.template.spec.containers[0].image' /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.XxnWG8ymb4 ++ mktemp + local LAST_ERR=/tmp/tmp.5tDZP5LJuF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + yq eval '(. | select(.kind == "Deployment") | .spec.template.spec.containers[0].image) = "perconalab/version-service:main-latest"' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XxnWG8ymb4 deployment.apps/version-service-cr created service/version-service-cr created + cat /tmp/tmp.5tDZP5LJuF + rm /tmp/tmp.XxnWG8ymb4 /tmp/tmp.5tDZP5LJuF + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.w3j3kKGyuX ++ mktemp + local LAST_ERR=/tmp/tmp.3P43IxiAbp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.w3j3kKGyuX pod "version-service-559c4df55c-579hg" deleted from smart-update2-6497 namespace + cat /tmp/tmp.3P43IxiAbp + rm /tmp/tmp.w3j3kKGyuX /tmp/tmp.3P43IxiAbp + return 0 ++ echo -n percona/percona-xtradb-cluster:8.4.5-5.1 ++ sed -r 's/^.*:([0-9]+.[0-9]+).*/\1/' + IMAGE_PREFIX=8.4 + desc 'Enable telemetry on operator level' + set +o xtrace ----------------------------------------------------------------------------------- Enable telemetry on operator level ----------------------------------------------------------------------------------- + kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "false"' ++ mktemp + kubectl_bin apply -n pxc-operator -f - + local LAST_OUT=/tmp/tmp.NtAUznERAv ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.HNFPZDkFzN + local exit_status=0 ++ seq 0 2 + local LAST_OUT=/tmp/tmp.eOcjzOkjrF ++ mktemp + for i in '$(seq 0 2)' + set +e + kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + local LAST_ERR=/tmp/tmp.ydJ1Q5w2pP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n pxc-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NtAUznERAv + cat /tmp/tmp.HNFPZDkFzN + rm /tmp/tmp.NtAUznERAv /tmp/tmp.HNFPZDkFzN + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eOcjzOkjrF deployment.apps/percona-xtradb-cluster-operator configured + cat /tmp/tmp.ydJ1Q5w2pP + rm /tmp/tmp.eOcjzOkjrF /tmp/tmp.ydJ1Q5w2pP + return 0 + sleep 30 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ head -1 ++ local LAST_OUT=/tmp/tmp.aNDGlYIoar +++ mktemp ++ local LAST_ERR=/tmp/tmp.2jgjaRVeLH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aNDGlYIoar ++ cat /tmp/tmp.2jgjaRVeLH ++ rm /tmp/tmp.aNDGlYIoar /tmp/tmp.2jgjaRVeLH ++ return 0 + wait_pod percona-xtradb-cluster-operator-75c5849bb-l8wdz 480 pxc-operator + local pod=percona-xtradb-cluster-operator-75c5849bb-l8wdz + local max_retry=480 + local ns=pxc-operator ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo percona-xtradb-cluster-operator-75c5849bb-l8wdz + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-75c5849bb-l8wdz condition met E0702 04:20:22.287850 10799 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-75c5849bb-l8wdz&resourceVersion=1782966021416531000&timeoutSeconds=515&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-75c5849bb-l8wdz to become Ready.Ok + check_telemetry_transfer http://version-service-cr.smart-update2-6497.svc.cluster.local:11000 disabled enabled + local cr_vs_uri=http://version-service-cr.smart-update2-6497.svc.cluster.local:11000 + local cr_vs_channel=disabled + local telemetry_state=enabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.VSdvjaIHAB ++ mktemp + local LAST_ERR=/tmp/tmp.qtRjWvKWY3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VSdvjaIHAB deployment.apps/pxc-client created + cat /tmp/tmp.qtRjWvKWY3 + rm /tmp/tmp.VSdvjaIHAB /tmp/tmp.qtRjWvKWY3 + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.uAInS79OVn ++ mktemp + local LAST_ERR=/tmp/tmp.ya2oRKogp9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uAInS79OVn secret/minimal-cluster created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.ya2oRKogp9 + rm /tmp/tmp.uAInS79OVn /tmp/tmp.ya2oRKogp9 + return 0 + yq eval '.spec.upgradeOptions.apply="disabled"' + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.4.5-5.1"' + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + yq eval '.spec.crVersion="9.9.9"' + yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"' + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858"' + kubectl_bin apply -f - + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-6497.svc.cluster.local:11000"' ++ mktemp + local LAST_OUT=/tmp/tmp.owmrrcMlFb ++ mktemp + local LAST_ERR=/tmp/tmp.JxwDtZ6p6X + local exit_status=0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/cr-minimal.yaml ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.owmrrcMlFb perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.JxwDtZ6p6X + rm /tmp/tmp.owmrrcMlFb /tmp/tmp.JxwDtZ6p6X + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-pxc ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-pxc ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zthSdbtp8y ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jl03EFCzhw +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.zthSdbtp8y +++ cat /tmp/tmp.jl03EFCzhw +++ rm /tmp/tmp.zthSdbtp8y /tmp/tmp.jl03EFCzhw +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-haproxy ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-haproxy ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo minimal-cluster-haproxy-0 + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met E0702 04:22:28.966357 21484 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dminimal-cluster-haproxy-0&resourceVersion=1782966145086328000&timeoutSeconds=305&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.e4W9ljbW6j +++ mktemp ++ local LAST_ERR=/tmp/tmp.gwlRQtn9bT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.e4W9ljbW6j ++ cat /tmp/tmp.gwlRQtn9bT ++ rm /tmp/tmp.e4W9ljbW6j /tmp/tmp.gwlRQtn9bT ++ return 0 + client_pod=pxc-client-67fc4995bb-zltbm + wait_pod pxc-client-67fc4995bb-zltbm + local pod=pxc-client-67fc4995bb-zltbm + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-zltbm ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-zltbm condition met E0702 04:22:40.493762 23165 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-zltbm&resourceVersion=1782966157789246000&timeoutSeconds=489&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-zltbm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YHapeyKkPt +++ mktemp ++ local LAST_ERR=/tmp/tmp.QOX7Oi4pLZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YHapeyKkPt ++ cat /tmp/tmp.QOX7Oi4pLZ ++ rm /tmp/tmp.YHapeyKkPt /tmp/tmp.QOX7Oi4pLZ ++ return 0 + client_pod=pxc-client-67fc4995bb-zltbm + wait_pod pxc-client-67fc4995bb-zltbm + local pod=pxc-client-67fc4995bb-zltbm + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-zltbm ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-zltbm condition met E0702 04:22:57.633138 25756 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-zltbm&resourceVersion=1782966175632663000&timeoutSeconds=597&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-zltbm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.start_time")' + grep -Eo '\{.*\}' + jq 'del(."grpc.time_ms")' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + grep -E 'server request payload|unary call' + jq 'del(."grpc.request.content".msg.kubeVersion)' + kubectl_bin logs version-service-cr-76d4659869-w42gr ++ mktemp + local LAST_OUT=/tmp/tmp.zhLROjFJYP ++ mktemp + local LAST_ERR=/tmp/tmp.hs2rt4WOKV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-76d4659869-w42gr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zhLROjFJYP + cat /tmp/tmp.hs2rt4WOKV + rm /tmp/tmp.zhLROjFJYP /tmp/tmp.hs2rt4WOKV + return 0 + jq 'del(."grpc.start_time")' + jq 'del(."grpc.time_ms")' + grep -E 'server request payload|unary call' + grep -Eo '\{.*\}' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.request.content".msg.kubeVersion)' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-559c4df55c-gtwhp ++ mktemp + local LAST_OUT=/tmp/tmp.ow7qDtwnih ++ mktemp + local LAST_ERR=/tmp/tmp.MsLufQ9uqn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-559c4df55c-gtwhp + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ow7qDtwnih + cat /tmp/tmp.MsLufQ9uqn + rm /tmp/tmp.ow7qDtwnih /tmp/tmp.MsLufQ9uqn + return 0 + local telemetry_log_file=enabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' disabled == disabled -a enabled == enabled ']' + desc 'operator fallback VS should have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator fallback VS should have telemetry ----------------------------------------------------------------------------------- + diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json /dev/fd/63 ++ grep -f /tmp/tmp.7fN8FXUd19/enabled_telemetry.version-service.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json + desc 'CR VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- CR VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.7fN8FXUd19/enabled_telemetry.version-service-cr.log.json ]] + local image_prefix=disabled + image_prefix=disabled + local telemetry_cr_log_file=enabled_telemetry.version-service-cr-disabled-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + [[ disabled == \d\i\s\a\b\l\e\d\-\r\e\c\o\m\m\e\n\d\e\d ]] + [[ disabled == \d\i\s\a\b\l\e\d\-\l\a\t\e\s\t ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' disabled == disabled -a enabled == disabled ']' + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.a39esSjCJs ++ mktemp + local LAST_ERR=/tmp/tmp.K2bMFyGm9J + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.a39esSjCJs perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.K2bMFyGm9J + rm /tmp/tmp.a39esSjCJs /tmp/tmp.K2bMFyGm9J + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ mktemp ++ local LAST_OUT=/tmp/tmp.l6SB7FyiTS +++ mktemp ++ local LAST_ERR=/tmp/tmp.rgpiWrF7gR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.l6SB7FyiTS ++ cat /tmp/tmp.rgpiWrF7gR ++ rm /tmp/tmp.l6SB7FyiTS /tmp/tmp.rgpiWrF7gR ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-75c5849bb-l8wdz ++ mktemp + local LAST_OUT=/tmp/tmp.ABI60CU3hq ++ mktemp + local LAST_ERR=/tmp/tmp.ORn3bQAWwb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-75c5849bb-l8wdz + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ABI60CU3hq pod "percona-xtradb-cluster-operator-75c5849bb-l8wdz" deleted from pxc-operator namespace + cat /tmp/tmp.ORn3bQAWwb + rm /tmp/tmp.ABI60CU3hq /tmp/tmp.ORn3bQAWwb + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.MBc3ZV7FOf ++ mktemp + local LAST_ERR=/tmp/tmp.p9bg0efLna + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MBc3ZV7FOf perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-6497 namespace + cat /tmp/tmp.p9bg0efLna + rm /tmp/tmp.MBc3ZV7FOf /tmp/tmp.p9bg0efLna + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.aq5iQqE8y6 ++ mktemp + local LAST_ERR=/tmp/tmp.Gp7QMfip3C + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.aq5iQqE8y6 deployment.apps "pxc-client" deleted from smart-update2-6497 namespace + cat /tmp/tmp.Gp7QMfip3C + rm /tmp/tmp.aq5iQqE8y6 /tmp/tmp.Gp7QMfip3C + return 0 + sleep 30 + desc 'Disabling telemetry on the operator level' + set +o xtrace ----------------------------------------------------------------------------------- Disabling telemetry on the operator level ----------------------------------------------------------------------------------- + kubectl_bin delete pod -l run=version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.pe2ISeESYl ++ mktemp + local LAST_ERR=/tmp/tmp.9EXoU6BhhJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pe2ISeESYl pod "version-service-cr-76d4659869-w42gr" deleted from smart-update2-6497 namespace + cat /tmp/tmp.9EXoU6BhhJ + rm /tmp/tmp.pe2ISeESYl /tmp/tmp.9EXoU6BhhJ + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.aTR0l1yh7l ++ mktemp + local LAST_ERR=/tmp/tmp.W0f9KZcnnZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.aTR0l1yh7l pod "version-service-559c4df55c-gtwhp" deleted from smart-update2-6497 namespace + cat /tmp/tmp.W0f9KZcnnZ + rm /tmp/tmp.aTR0l1yh7l /tmp/tmp.W0f9KZcnnZ + return 0 + kubectl_bin apply -n pxc-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.IPz6mCU09G ++ mktemp + local LAST_ERR=/tmp/tmp.fe3y3XgooD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n pxc-operator -f - + kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' ++ mktemp + local LAST_OUT=/tmp/tmp.BnUpgYBisk ++ mktemp + local LAST_ERR=/tmp/tmp.qP7uEcj6Pv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BnUpgYBisk + cat /tmp/tmp.qP7uEcj6Pv + rm /tmp/tmp.BnUpgYBisk /tmp/tmp.qP7uEcj6Pv + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IPz6mCU09G deployment.apps/percona-xtradb-cluster-operator configured + cat /tmp/tmp.fe3y3XgooD + rm /tmp/tmp.IPz6mCU09G /tmp/tmp.fe3y3XgooD + return 0 + sleep 30 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Hc0lilJb45 +++ mktemp ++ local LAST_ERR=/tmp/tmp.fHORJd2gts ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Hc0lilJb45 ++ cat /tmp/tmp.fHORJd2gts ++ rm /tmp/tmp.Hc0lilJb45 /tmp/tmp.fHORJd2gts ++ return 0 + wait_pod percona-xtradb-cluster-operator-b7b8b9959-q55pt 480 pxc-operator + local pod=percona-xtradb-cluster-operator-b7b8b9959-q55pt + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-b7b8b9959-q55pt ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-b7b8b9959-q55pt condition met E0702 04:24:57.943076 10629 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-b7b8b9959-q55pt&resourceVersion=1782966296384921000&timeoutSeconds=412&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-b7b8b9959-q55pt to become Ready.Ok + check_telemetry_transfer http://version-service-cr.smart-update2-6497.svc.cluster.local:11000 8.4-latest disabled + local cr_vs_uri=http://version-service-cr.smart-update2-6497.svc.cluster.local:11000 + local cr_vs_channel=8.4-latest + local telemetry_state=disabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.y9qQ6sEcjJ ++ mktemp + local LAST_ERR=/tmp/tmp.ipnDetBnrU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.y9qQ6sEcjJ deployment.apps/pxc-client created + cat /tmp/tmp.ipnDetBnrU + rm /tmp/tmp.y9qQ6sEcjJ /tmp/tmp.ipnDetBnrU + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + kubectl_bin apply -f - + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' ++ mktemp + local LAST_OUT=/tmp/tmp.YAWm5sa3wi ++ mktemp + local LAST_ERR=/tmp/tmp.ylbVtM4Egp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YAWm5sa3wi secret/minimal-cluster unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.ylbVtM4Egp + rm /tmp/tmp.YAWm5sa3wi /tmp/tmp.ylbVtM4Egp + return 0 + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-6497.svc.cluster.local:11000"' + yq eval '.spec.upgradeOptions.apply="8.4-latest"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/cr-minimal.yaml + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.4.5-5.1"' + yq eval '.spec.crVersion="9.9.9"' + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858"' + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + kubectl_bin apply -f - + yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"' ++ mktemp + local LAST_OUT=/tmp/tmp.jEgFegixJ4 ++ mktemp + local LAST_ERR=/tmp/tmp.TXbQSNkMsW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jEgFegixJ4 perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.TXbQSNkMsW + rm /tmp/tmp.jEgFegixJ4 /tmp/tmp.TXbQSNkMsW + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-pxc ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-pxc ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.YxPYCTTori ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ktt7qlX85Z +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.YxPYCTTori +++ cat /tmp/tmp.ktt7qlX85Z +++ rm /tmp/tmp.YxPYCTTori /tmp/tmp.ktt7qlX85Z +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-haproxy ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-haproxy ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo minimal-cluster-haproxy-0 ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met E0702 04:27:14.827014 29975 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dminimal-cluster-haproxy-0&resourceVersion=1782966432225534000&timeoutSeconds=308&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nHq8xpLxuL +++ mktemp ++ local LAST_ERR=/tmp/tmp.mDITAk6nAZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nHq8xpLxuL ++ cat /tmp/tmp.mDITAk6nAZ ++ rm /tmp/tmp.nHq8xpLxuL /tmp/tmp.mDITAk6nAZ ++ return 0 + client_pod=pxc-client-67fc4995bb-w5s5p + wait_pod pxc-client-67fc4995bb-w5s5p + local pod=pxc-client-67fc4995bb-w5s5p + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-w5s5p + local container= + set +o xtrace pod/pxc-client-67fc4995bb-w5s5p condition met E0702 04:27:24.829505 31879 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w5s5p&resourceVersion=1782966443773661000&timeoutSeconds=569&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-w5s5p to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BIJbDXTfeO +++ mktemp ++ local LAST_ERR=/tmp/tmp.ikt1tvWT8M ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BIJbDXTfeO ++ cat /tmp/tmp.ikt1tvWT8M ++ rm /tmp/tmp.BIJbDXTfeO /tmp/tmp.ikt1tvWT8M ++ return 0 + client_pod=pxc-client-67fc4995bb-w5s5p + wait_pod pxc-client-67fc4995bb-w5s5p + local pod=pxc-client-67fc4995bb-w5s5p + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-w5s5p ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-w5s5p condition met E0702 04:27:48.800424 2867 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w5s5p&resourceVersion=1782966465223174000&timeoutSeconds=418&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-w5s5p to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + grep -E 'server request payload|unary call' + jq 'del(."grpc.request.content".msg.kubeVersion)' + grep -Eo '\{.*\}' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + jq 'del(."grpc.time_ms")' + kubectl_bin logs version-service-cr-76d4659869-zngnk ++ mktemp + local LAST_OUT=/tmp/tmp.EfhVWFqUGS ++ mktemp + local LAST_ERR=/tmp/tmp.PcW5bnDr3v + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-76d4659869-zngnk + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EfhVWFqUGS + cat /tmp/tmp.PcW5bnDr3v + rm /tmp/tmp.EfhVWFqUGS /tmp/tmp.PcW5bnDr3v + return 0 + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.request.content".msg.kubeVersion)' + grep -E 'server request payload|unary call' + grep -Eo '\{.*\}' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.time_ms")' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-559c4df55c-s6lq6 ++ mktemp + local LAST_OUT=/tmp/tmp.z8wyQGadoA ++ mktemp + local LAST_ERR=/tmp/tmp.KvGFFcmXa5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-559c4df55c-s6lq6 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.z8wyQGadoA + cat /tmp/tmp.KvGFFcmXa5 + rm /tmp/tmp.z8wyQGadoA /tmp/tmp.KvGFFcmXa5 + return 0 + local telemetry_log_file=disabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' 8.4-latest == disabled -a disabled == enabled ']' + local image_prefix=8.4-latest + image_prefix=8.4 + local telemetry_cr_log_file=disabled_telemetry.version-service-cr-8.4-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + [[ 8.4-latest == \8\.\4\-\r\e\c\o\m\m\e\n\d\e\d ]] + [[ 8.4-latest == \8\.\4\-\l\a\t\e\s\t ]] + '[' disabled == disabled ']' + desc 'cr VS should have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- cr VS should have telemetry ----------------------------------------------------------------------------------- + diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.4-cw.log.json /dev/fd/63 ++ grep -f /tmp/tmp.7fN8FXUd19/disabled_telemetry.version-service-cr.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.4-cw.log.json + desc 'operator VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.7fN8FXUd19/disabled_telemetry.version-service.log.json ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' 8.4-latest == disabled -a disabled == disabled ']' + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.nSgnnO5Suk ++ mktemp + local LAST_ERR=/tmp/tmp.ZEgtfVYQEZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nSgnnO5Suk perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.ZEgtfVYQEZ + rm /tmp/tmp.nSgnnO5Suk /tmp/tmp.ZEgtfVYQEZ + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ mktemp ++ head -1 ++ local LAST_OUT=/tmp/tmp.EtGRhI1L1o +++ mktemp ++ local LAST_ERR=/tmp/tmp.T4dXIVAeCW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EtGRhI1L1o ++ cat /tmp/tmp.T4dXIVAeCW ++ rm /tmp/tmp.EtGRhI1L1o /tmp/tmp.T4dXIVAeCW ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-b7b8b9959-q55pt ++ mktemp + local LAST_OUT=/tmp/tmp.acoQY1Yoia ++ mktemp + local LAST_ERR=/tmp/tmp.CQrkmU14gC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-b7b8b9959-q55pt + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.acoQY1Yoia pod "percona-xtradb-cluster-operator-b7b8b9959-q55pt" deleted from pxc-operator namespace + cat /tmp/tmp.CQrkmU14gC + rm /tmp/tmp.acoQY1Yoia /tmp/tmp.CQrkmU14gC + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.hws9TpVZlL ++ mktemp + local LAST_ERR=/tmp/tmp.XLDCJVVURR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hws9TpVZlL perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-6497 namespace + cat /tmp/tmp.XLDCJVVURR + rm /tmp/tmp.hws9TpVZlL /tmp/tmp.XLDCJVVURR + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.YHnrXUMQRU ++ mktemp + local LAST_ERR=/tmp/tmp.R9D6FNqeOB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YHnrXUMQRU deployment.apps "pxc-client" deleted from smart-update2-6497 namespace + cat /tmp/tmp.R9D6FNqeOB + rm /tmp/tmp.YHnrXUMQRU /tmp/tmp.R9D6FNqeOB + return 0 + sleep 30 + kubectl_bin delete pod -l run=version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.nUFXVPGcxj ++ mktemp + local LAST_ERR=/tmp/tmp.7V7UiOoFjP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nUFXVPGcxj pod "version-service-cr-76d4659869-zngnk" deleted from smart-update2-6497 namespace + cat /tmp/tmp.7V7UiOoFjP + rm /tmp/tmp.nUFXVPGcxj /tmp/tmp.7V7UiOoFjP + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.whGbK6JIbd ++ mktemp + local LAST_ERR=/tmp/tmp.HAgP2N5OOJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.whGbK6JIbd pod "version-service-559c4df55c-s6lq6" deleted from smart-update2-6497 namespace + cat /tmp/tmp.HAgP2N5OOJ + rm /tmp/tmp.whGbK6JIbd /tmp/tmp.HAgP2N5OOJ + return 0 + check_telemetry_transfer http://version-service-cr.smart-update2-6497.svc.cluster.local:11000 disabled disabled + local cr_vs_uri=http://version-service-cr.smart-update2-6497.svc.cluster.local:11000 + local cr_vs_channel=disabled + local telemetry_state=disabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.huIKplShgs ++ mktemp + local LAST_ERR=/tmp/tmp.7cKtx3bIQU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.huIKplShgs deployment.apps/pxc-client created + cat /tmp/tmp.7cKtx3bIQU + rm /tmp/tmp.huIKplShgs /tmp/tmp.7cKtx3bIQU + return 0 + kubectl_bin apply -f - + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.epwiyRRD7F ++ mktemp + local LAST_ERR=/tmp/tmp.WIR9fWerJD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.epwiyRRD7F secret/minimal-cluster unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.WIR9fWerJD + rm /tmp/tmp.epwiyRRD7F /tmp/tmp.WIR9fWerJD + return 0 + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-6497.svc.cluster.local:11000"' + kubectl_bin apply -f - + yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"' + yq eval '.spec.crVersion="9.9.9"' + yq eval '.spec.upgradeOptions.apply="disabled"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/cr-minimal.yaml + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.4.5-5.1"' ++ mktemp + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858"' + local LAST_OUT=/tmp/tmp.NMoUb0cHa6 ++ mktemp + local LAST_ERR=/tmp/tmp.QoMlxGQjGY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NMoUb0cHa6 perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.QoMlxGQjGY + rm /tmp/tmp.NMoUb0cHa6 /tmp/tmp.QoMlxGQjGY + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-pxc ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-pxc ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo minimal-cluster-pxc-0 + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LRCJqLb12w ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RiWQKeQ9jF +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.LRCJqLb12w +++ cat /tmp/tmp.RiWQKeQ9jF +++ rm /tmp/tmp.LRCJqLb12w /tmp/tmp.RiWQKeQ9jF +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-haproxy ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-haproxy ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo minimal-cluster-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met E0702 04:31:51.437585 6928 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dminimal-cluster-haproxy-0&resourceVersion=1782966710230539000&timeoutSeconds=590&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Y4oi0JdX7Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.H8Ji50LNEu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Y4oi0JdX7Z ++ cat /tmp/tmp.H8Ji50LNEu ++ rm /tmp/tmp.Y4oi0JdX7Z /tmp/tmp.H8Ji50LNEu ++ return 0 + client_pod=pxc-client-67fc4995bb-frz7b + wait_pod pxc-client-67fc4995bb-frz7b + local pod=pxc-client-67fc4995bb-frz7b + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-frz7b ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-frz7b condition met E0702 04:31:59.390589 8162 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-frz7b&resourceVersion=1782966717746498000&timeoutSeconds=560&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-frz7b to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ExwJgv3qe9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.W1yiRbJ5v6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ExwJgv3qe9 ++ cat /tmp/tmp.W1yiRbJ5v6 ++ rm /tmp/tmp.ExwJgv3qe9 /tmp/tmp.W1yiRbJ5v6 ++ return 0 + client_pod=pxc-client-67fc4995bb-frz7b + wait_pod pxc-client-67fc4995bb-frz7b + local pod=pxc-client-67fc4995bb-frz7b + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-frz7b ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-frz7b condition met E0702 04:32:11.180967 9831 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-frz7b&resourceVersion=1782966729419486000&timeoutSeconds=558&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-frz7b to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + grep -E 'server request payload|unary call' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.time_ms")' + grep -Eo '\{.*\}' + kubectl_bin logs version-service-cr-76d4659869-flw8c ++ mktemp + local LAST_OUT=/tmp/tmp.28bJMCMA5D ++ mktemp + local LAST_ERR=/tmp/tmp.ebJiy0KkQ9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-76d4659869-flw8c + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.28bJMCMA5D + cat /tmp/tmp.ebJiy0KkQ9 + rm /tmp/tmp.28bJMCMA5D /tmp/tmp.ebJiy0KkQ9 + return 0 ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.request.content".msg.kubeVersion)' + grep -Eo '\{.*\}' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + grep -E 'server request payload|unary call' + kubectl_bin logs version-service-559c4df55c-k4mzm ++ mktemp + local LAST_OUT=/tmp/tmp.ITmIPXi6re ++ mktemp + local LAST_ERR=/tmp/tmp.tznN94yW68 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-559c4df55c-k4mzm + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ITmIPXi6re + cat /tmp/tmp.tznN94yW68 + rm /tmp/tmp.ITmIPXi6re /tmp/tmp.tznN94yW68 + return 0 + local telemetry_log_file=disabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' disabled == disabled -a disabled == enabled ']' + local image_prefix=disabled + image_prefix=disabled + local telemetry_cr_log_file=disabled_telemetry.version-service-cr-disabled-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + [[ disabled == \d\i\s\a\b\l\e\d\-\r\e\c\o\m\m\e\n\d\e\d ]] + [[ disabled == \d\i\s\a\b\l\e\d\-\l\a\t\e\s\t ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' disabled == disabled -a disabled == disabled ']' + desc 'CR VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- CR VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.7fN8FXUd19/disabled_telemetry.version-service-cr.log.json ]] + desc 'operator VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.7fN8FXUd19/disabled_telemetry.version-service.log.json ]] + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.zuH5Nrbib8 ++ mktemp + local LAST_ERR=/tmp/tmp.nNlqsjPQuo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zuH5Nrbib8 perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.nNlqsjPQuo + rm /tmp/tmp.zuH5Nrbib8 /tmp/tmp.nNlqsjPQuo + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ local LAST_OUT=/tmp/tmp.1tEMYJAIJ6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.NnKJJ0xW2D ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1tEMYJAIJ6 ++ cat /tmp/tmp.NnKJJ0xW2D ++ rm /tmp/tmp.1tEMYJAIJ6 /tmp/tmp.NnKJJ0xW2D ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-b7b8b9959-djlkt ++ mktemp + local LAST_OUT=/tmp/tmp.U2QMYhnLkM ++ mktemp + local LAST_ERR=/tmp/tmp.QrZrTvkDZ7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-b7b8b9959-djlkt + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.U2QMYhnLkM pod "percona-xtradb-cluster-operator-b7b8b9959-djlkt" deleted from pxc-operator namespace + cat /tmp/tmp.QrZrTvkDZ7 + rm /tmp/tmp.U2QMYhnLkM /tmp/tmp.QrZrTvkDZ7 + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.kAzgrf5Fq2 ++ mktemp + local LAST_ERR=/tmp/tmp.fIy1PW5hqK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kAzgrf5Fq2 perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-6497 namespace + cat /tmp/tmp.fIy1PW5hqK + rm /tmp/tmp.kAzgrf5Fq2 /tmp/tmp.fIy1PW5hqK + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.kTsi1DYp6p ++ mktemp + local LAST_ERR=/tmp/tmp.pKxZyCz8Dt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kTsi1DYp6p deployment.apps "pxc-client" deleted from smart-update2-6497 namespace + cat /tmp/tmp.pKxZyCz8Dt + rm /tmp/tmp.kTsi1DYp6p /tmp/tmp.pKxZyCz8Dt + return 0 + sleep 30 + kubectl_bin delete deployment version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.JR2tAfyuQZ ++ mktemp + local LAST_ERR=/tmp/tmp.yjopTK1vW3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deployment version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JR2tAfyuQZ deployment.apps "version-service-cr" deleted from smart-update2-6497 namespace + cat /tmp/tmp.yjopTK1vW3 + rm /tmp/tmp.JR2tAfyuQZ /tmp/tmp.yjopTK1vW3 + return 0 + desc 'Telemetry testing finished' + set +o xtrace ----------------------------------------------------------------------------------- Telemetry testing finished ----------------------------------------------------------------------------------- + desc 'PXC cluster with version service offline' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster with version service offline ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/smart-update-version-service-unreachable.yml /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858"' /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + spinup_pxc smart-update /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.TYbRhjJEKI ++ mktemp + local LAST_ERR=/tmp/tmp.zsAaint4wz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TYbRhjJEKI secret/my-cluster-secrets created secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.zsAaint4wz + rm /tmp/tmp.TYbRhjJEKI /tmp/tmp.zsAaint4wz + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.U5FoF6zLMJ + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-6497~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.FePUUJmsjT + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.U5FoF6zLMJ deployment.apps/pxc-client created + cat /tmp/tmp.FePUUJmsjT + rm /tmp/tmp.U5FoF6zLMJ /tmp/tmp.FePUUJmsjT + return 0 + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + apply_config /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local config_file=/tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local pvc_name= + '[' -z '' ']' + cat_config /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml '' + local input_file=/tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local pvc_name= + kubectl_bin apply -f - + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-6497~ + local LAST_OUT=/tmp/tmp.cfJMj77zJR + cat /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + local LAST_ERR=/tmp/tmp.FGEQPVumDk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cfJMj77zJR perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.FGEQPVumDk + rm /tmp/tmp.cfJMj77zJR /tmp/tmp.FGEQPVumDk + return 0 + desc 'waiting pods to be running for smart-update' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for smart-update ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UzViO1q3D0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OggGI3mnmS +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.UzViO1q3D0 +++ cat /tmp/tmp.OggGI3mnmS +++ rm /tmp/tmp.UzViO1q3D0 /tmp/tmp.OggGI3mnmS +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy ++ get_proxy_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UceCU3cfsl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.khYtjuggBV +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.UceCU3cfsl +++ cat /tmp/tmp.khYtjuggBV +++ rm /tmp/tmp.UceCU3cfsl /tmp/tmp.khYtjuggBV +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hizEGWI8CS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RYT5Bm3yUl +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.hizEGWI8CS +++ cat /tmp/tmp.RYT5Bm3yUl +++ rm /tmp/tmp.hizEGWI8CS /tmp/tmp.RYT5Bm3yUl +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z smart-update-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for smart-update-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for smart-update-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + wait_for_running smart-update-haproxy 2 + local name=smart-update-haproxy + local size=2 + local last_pod + local max_retry=480 + [[ -z smart-update-haproxy ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for smart-update-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for smart-update-haproxy ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met E0702 04:37:53.391726 26021 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-0&resourceVersion=1782967071711112000&timeoutSeconds=438&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-1 480 + local pod=smart-update-haproxy-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-haproxy-1 + local container= + set +o xtrace pod/smart-update-haproxy-1 condition met E0702 04:38:06.399250 27808 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-1&resourceVersion=1782967081712336000&timeoutSeconds=570&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-1 to become Ready.Ok + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rFysLukQX7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Vh0bHGpu8X ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rFysLukQX7 ++ cat /tmp/tmp.Vh0bHGpu8X ++ rm /tmp/tmp.rFysLukQX7 /tmp/tmp.Vh0bHGpu8X ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n5g23GfKqK +++ mktemp ++ local LAST_ERR=/tmp/tmp.sy7RRzmBeJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.n5g23GfKqK ++ cat /tmp/tmp.sy7RRzmBeJ ++ rm /tmp/tmp.n5g23GfKqK /tmp/tmp.sy7RRzmBeJ ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.HyVt9hEnpR ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.v9wLsrbVqL +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.HyVt9hEnpR +++++ cat /tmp/tmp.v9wLsrbVqL +++++ rm /tmp/tmp.HyVt9hEnpR /tmp/tmp.v9wLsrbVqL +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NOeJ4hFJld +++ mktemp ++ local LAST_ERR=/tmp/tmp.m8nSxqHBmh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NOeJ4hFJld ++ cat /tmp/tmp.m8nSxqHBmh ++ rm /tmp/tmp.NOeJ4hFJld /tmp/tmp.m8nSxqHBmh ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.yS8pdH5Bsp +++ mktemp ++ local LAST_ERR=/tmp/tmp.FmljOBkKzv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yS8pdH5Bsp ++ cat /tmp/tmp.FmljOBkKzv ++ rm /tmp/tmp.yS8pdH5Bsp /tmp/tmp.FmljOBkKzv ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vvW7HR94tL +++ mktemp ++ local LAST_ERR=/tmp/tmp.NjB5Z3UWAT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vvW7HR94tL ++ cat /tmp/tmp.NjB5Z3UWAT ++ rm /tmp/tmp.vvW7HR94tL /tmp/tmp.NjB5Z3UWAT ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-rssdb ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:38:55.001932 2766 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967132200566000&timeoutSeconds=504&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data smart-update 100500 no-proxy -P3306 + local cluster=smart-update + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ZxXKAHKn38 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nQiPlCrrex +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.ZxXKAHKn38 +++ cat /tmp/tmp.nQiPlCrrex +++ rm /tmp/tmp.ZxXKAHKn38 /tmp/tmp.nQiPlCrrex +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.a8lxpKmiKx +++ mktemp ++ local LAST_ERR=/tmp/tmp.zMPghc4mp6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.a8lxpKmiKx ++ cat /tmp/tmp.zMPghc4mp6 ++ rm /tmp/tmp.a8lxpKmiKx /tmp/tmp.zMPghc4mp6 ++ return 0 + root_pass=root_password ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JniQxCwDpz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9R49IFCflY +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.JniQxCwDpz +++ cat /tmp/tmp.9R49IFCflY +++ rm /tmp/tmp.JniQxCwDpz /tmp/tmp.9R49IFCflY +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + proxy=smart-update-haproxy + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h smart-update-pxc-0.smart-update-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dSaWtOTrMC +++ mktemp ++ local LAST_ERR=/tmp/tmp.PsSi4aMIez ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dSaWtOTrMC ++ cat /tmp/tmp.PsSi4aMIez ++ rm /tmp/tmp.dSaWtOTrMC /tmp/tmp.PsSi4aMIez ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-rssdb ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:39:11.409235 5431 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967149436890000&timeoutSeconds=397&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data smart-update select-1 no-proxy -P3306 + local cluster=smart-update + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.85hdgDIk9U ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UpAk8iDJsa +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.85hdgDIk9U +++ cat /tmp/tmp.UpAk8iDJsa +++ rm /tmp/tmp.85hdgDIk9U /tmp/tmp.UpAk8iDJsa +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Nn0AI4q2cU +++ mktemp ++ local LAST_ERR=/tmp/tmp.YQPlHf4jmL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Nn0AI4q2cU ++ cat /tmp/tmp.YQPlHf4jmL ++ rm /tmp/tmp.Nn0AI4q2cU /tmp/tmp.YQPlHf4jmL ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yCsFuGCnyt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2JqekKc2yQ +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.yCsFuGCnyt +++ cat /tmp/tmp.2JqekKc2yQ +++ rm /tmp/tmp.yCsFuGCnyt /tmp/tmp.2JqekKc2yQ +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kKNZvHcr2v +++ mktemp ++ local LAST_ERR=/tmp/tmp.RJIDF0nGTM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kKNZvHcr2v ++ cat /tmp/tmp.RJIDF0nGTM ++ rm /tmp/tmp.kKNZvHcr2v /tmp/tmp.RJIDF0nGTM ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-rssdb + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:39:56.038894 11849 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967193917712000&timeoutSeconds=466&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PhnPiFmH8w +++ mktemp ++ local LAST_ERR=/tmp/tmp.DKvhCHEWYj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PhnPiFmH8w ++ cat /tmp/tmp.DKvhCHEWYj ++ rm /tmp/tmp.PhnPiFmH8w /tmp/tmp.DKvhCHEWYj ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-58956d9c99-rssdb ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:40:03.827323 13175 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967202851046000&timeoutSeconds=307&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PAwhhFxMBM +++ mktemp ++ local LAST_ERR=/tmp/tmp.ashMfkbjsR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PAwhhFxMBM ++ cat /tmp/tmp.ashMfkbjsR ++ rm /tmp/tmp.PAwhhFxMBM /tmp/tmp.ashMfkbjsR ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-rssdb + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:40:12.196863 14377 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967211382051000&timeoutSeconds=326&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + kubectl exec smart-update-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my + return 1 + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ngb0Dn4PDh +++ mktemp ++ local LAST_ERR=/tmp/tmp.IrADWd1U71 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ngb0Dn4PDh ++ cat /tmp/tmp.IrADWd1U71 ++ rm /tmp/tmp.Ngb0Dn4PDh /tmp/tmp.IrADWd1U71 ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.46JixLHTZl +++ mktemp ++ local LAST_ERR=/tmp/tmp.u5jVIyX8vJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.46JixLHTZl ++ cat /tmp/tmp.u5jVIyX8vJ ++ rm /tmp/tmp.46JixLHTZl /tmp/tmp.u5jVIyX8vJ ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.rJKV4PvQ5Z ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.3fycm8AcMu +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.rJKV4PvQ5Z +++++ cat /tmp/tmp.3fycm8AcMu +++++ rm /tmp/tmp.rJKV4PvQ5Z /tmp/tmp.3fycm8AcMu +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.koJOdRMfLX +++ mktemp ++ local LAST_ERR=/tmp/tmp.xuFL3iNn8W ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.koJOdRMfLX ++ cat /tmp/tmp.xuFL3iNn8W ++ rm /tmp/tmp.koJOdRMfLX /tmp/tmp.xuFL3iNn8W ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ kubectl_bin get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cJ1oxahNZn +++ mktemp ++ local LAST_ERR=/tmp/tmp.5zGipbrrxU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cJ1oxahNZn ++ cat /tmp/tmp.5zGipbrrxU ++ rm /tmp/tmp.cJ1oxahNZn /tmp/tmp.5zGipbrrxU ++ return 0 + [[ percona/percona-xtradb-cluster:8.4.5-5.1 != \p\e\r\c\o\n\a\/\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\:\8\.\4\.\5\-\5\.\1 ]] + desc 'PXC cluster update with latest image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with latest image by version service ----------------------------------------------------------------------------------- + vs_image=latest ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PjEyks4Ltg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ANeoVE0av2 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.PjEyks4Ltg +++ cat /tmp/tmp.ANeoVE0av2 +++ rm /tmp/tmp.PjEyks4Ltg /tmp/tmp.ANeoVE0av2 +++ return 0 ++ client_pod=pxc-client-58956d9c99-rssdb ++ wait_pod pxc-client-58956d9c99-rssdb ++ local pod=pxc-client-58956d9c99-rssdb ++ local max_retry=480 ++ local ns= +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ grep -E '^(pxc|proxysql)$' +++ echo pxc-client-58956d9c99-rssdb ++ local container= ++ set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:40:36.918567 17816 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967235028326000&timeoutSeconds=555&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.TQs61PzyF8 ++ mktemp + local LAST_ERR=/tmp/tmp.XrsyB5OAmU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TQs61PzyF8 perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.XrsyB5OAmU + rm /tmp/tmp.TQs61PzyF8 /tmp/tmp.XrsyB5OAmU + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DiXCWTdzIt +++ mktemp ++ local LAST_ERR=/tmp/tmp.rEyjHJAzgt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DiXCWTdzIt ++ cat /tmp/tmp.rEyjHJAzgt ++ rm /tmp/tmp.DiXCWTdzIt /tmp/tmp.rEyjHJAzgt ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GbnvCv3mgE +++ mktemp ++ local LAST_ERR=/tmp/tmp.lnFXYeFm67 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GbnvCv3mgE ++ cat /tmp/tmp.lnFXYeFm67 ++ rm /tmp/tmp.GbnvCv3mgE /tmp/tmp.lnFXYeFm67 ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Mt2u1ihutg ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.1Mm2N1W26p +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Mt2u1ihutg +++++ cat /tmp/tmp.1Mm2N1W26p +++++ rm /tmp/tmp.Mt2u1ihutg /tmp/tmp.1Mm2N1W26p +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9tRnMl5XsX +++ mktemp ++ local LAST_ERR=/tmp/tmp.knWCSyuPDV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9tRnMl5XsX ++ cat /tmp/tmp.knWCSyuPDV ++ rm /tmp/tmp.9tRnMl5XsX /tmp/tmp.knWCSyuPDV ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.r3K52skBFJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.aRZEfntHFj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.r3K52skBFJ ++ cat /tmp/tmp.aRZEfntHFj ++ rm /tmp/tmp.r3K52skBFJ /tmp/tmp.aRZEfntHFj ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-rssdb ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:41:57.413029 29546 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967315321973000&timeoutSeconds=493&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nJisjSgkcw +++ mktemp ++ local LAST_ERR=/tmp/tmp.eb826Xc3yD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nJisjSgkcw ++ cat /tmp/tmp.eb826Xc3yD ++ rm /tmp/tmp.nJisjSgkcw /tmp/tmp.eb826Xc3yD ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-rssdb ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:42:09.414039 31332 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967326899864000&timeoutSeconds=310&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gjlOlVwOIP +++ mktemp ++ local LAST_ERR=/tmp/tmp.soYFb4gqgg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gjlOlVwOIP ++ cat /tmp/tmp.soYFb4gqgg ++ rm /tmp/tmp.gjlOlVwOIP /tmp/tmp.soYFb4gqgg ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-rssdb + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:42:24.638969 1304 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967342409272000&timeoutSeconds=525&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + kubectl_bin delete -f /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.X73tlHAs9x ++ mktemp + local LAST_ERR=/tmp/tmp.N9uVr3SMM9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.X73tlHAs9x perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-6497 namespace + cat /tmp/tmp.N9uVr3SMM9 + rm /tmp/tmp.X73tlHAs9x /tmp/tmp.N9uVr3SMM9 + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.067TOGMkEI ++ mktemp + local LAST_ERR=/tmp/tmp.tc0hVEw7O7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.067TOGMkEI persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-6497 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-6497 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-6497 namespace + cat /tmp/tmp.tc0hVEw7O7 + rm /tmp/tmp.067TOGMkEI /tmp/tmp.tc0hVEw7O7 + return 0 + desc 'PXC cluster update with the latest image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with the latest image by version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.chvNv3Zygb ++ mktemp + local LAST_ERR=/tmp/tmp.YoDdinBOBv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.chvNv3Zygb secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.YoDdinBOBv + rm /tmp/tmp.chvNv3Zygb /tmp/tmp.YoDdinBOBv + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml '' + kubectl_bin apply -f - + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-6497~ + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.8KkJlbEeoj + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + local LAST_ERR=/tmp/tmp.0JOPloq8ct + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8KkJlbEeoj deployment.apps/pxc-client unchanged + cat /tmp/tmp.0JOPloq8ct + rm /tmp/tmp.8KkJlbEeoj /tmp/tmp.0JOPloq8ct + return 0 + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + apply_config /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local config_file=/tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml '' + local input_file=/tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-6497~ + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + local LAST_OUT=/tmp/tmp.H9zWNU5jEV + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' ++ mktemp + local LAST_ERR=/tmp/tmp.DOgoGRQEwM + local exit_status=0 + cat /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.H9zWNU5jEV perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.DOgoGRQEwM + rm /tmp/tmp.H9zWNU5jEV /tmp/tmp.DOgoGRQEwM + return 0 + desc 'waiting pods to be running for smart-update' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for smart-update ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mbjZ3WcKeQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zVYB9hHgMJ +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.mbjZ3WcKeQ +++ cat /tmp/tmp.zVYB9hHgMJ +++ rm /tmp/tmp.mbjZ3WcKeQ /tmp/tmp.zVYB9hHgMJ +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy ++ get_proxy_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.FdoPlKE1BE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cxRzfmYJht +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.FdoPlKE1BE +++ cat /tmp/tmp.cxRzfmYJht +++ rm /tmp/tmp.FdoPlKE1BE /tmp/tmp.cxRzfmYJht +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.36up74u0KY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2yylgm0GDW +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.36up74u0KY +++ cat /tmp/tmp.2yylgm0GDW +++ rm /tmp/tmp.36up74u0KY /tmp/tmp.2yylgm0GDW +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z smart-update-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for smart-update-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for smart-update-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-2 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + wait_for_running smart-update-haproxy 2 + local name=smart-update-haproxy + local size=2 + local last_pod + local max_retry=480 + [[ -z smart-update-haproxy ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for smart-update-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for smart-update-haproxy ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-haproxy-0 + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met E0702 04:47:51.520724 17813 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-0&resourceVersion=1782967667650290000&timeoutSeconds=593&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-1 480 + local pod=smart-update-haproxy-1 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-1 condition met E0702 04:48:03.451762 19640 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-1&resourceVersion=1782967682650455000&timeoutSeconds=586&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-1 to become Ready.Ok + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YpiNzGJ8BB +++ mktemp ++ local LAST_ERR=/tmp/tmp.0jJFKnpsFG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YpiNzGJ8BB ++ cat /tmp/tmp.0jJFKnpsFG ++ rm /tmp/tmp.YpiNzGJ8BB /tmp/tmp.0jJFKnpsFG ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VKRR42mQ2Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.IeKI3nyttf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VKRR42mQ2Q ++ cat /tmp/tmp.IeKI3nyttf ++ rm /tmp/tmp.VKRR42mQ2Q /tmp/tmp.IeKI3nyttf ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.jaeATwk9u0 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.rnDqxlW6m0 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.jaeATwk9u0 +++++ cat /tmp/tmp.rnDqxlW6m0 +++++ rm /tmp/tmp.jaeATwk9u0 /tmp/tmp.rnDqxlW6m0 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aY2hc1RNux +++ mktemp ++ local LAST_ERR=/tmp/tmp.GTL0ceAAM5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aY2hc1RNux ++ cat /tmp/tmp.GTL0ceAAM5 ++ rm /tmp/tmp.aY2hc1RNux /tmp/tmp.GTL0ceAAM5 ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6dSSKY7aOS +++ mktemp ++ local LAST_ERR=/tmp/tmp.yMsyTTyuFl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6dSSKY7aOS ++ cat /tmp/tmp.yMsyTTyuFl ++ rm /tmp/tmp.6dSSKY7aOS /tmp/tmp.yMsyTTyuFl ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2OZoS31z6b +++ mktemp ++ local LAST_ERR=/tmp/tmp.qqaZ03Eq7Z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2OZoS31z6b ++ cat /tmp/tmp.qqaZ03Eq7Z ++ rm /tmp/tmp.2OZoS31z6b /tmp/tmp.qqaZ03Eq7Z ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-rssdb ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:48:42.597948 25695 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967720768809000&timeoutSeconds=468&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data smart-update 100500 no-proxy -P3306 + local cluster=smart-update + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.H3MiGdvnmh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gV4PCLja0z +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.H3MiGdvnmh +++ cat /tmp/tmp.gV4PCLja0z +++ rm /tmp/tmp.H3MiGdvnmh /tmp/tmp.gV4PCLja0z +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KSHeVIzmzZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.KmpwdJqTpi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KSHeVIzmzZ ++ cat /tmp/tmp.KmpwdJqTpi ++ rm /tmp/tmp.KSHeVIzmzZ /tmp/tmp.KmpwdJqTpi ++ return 0 + root_pass=root_password ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qrXReBJCui ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jsiW8bhVw3 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.qrXReBJCui +++ cat /tmp/tmp.jsiW8bhVw3 +++ rm /tmp/tmp.qrXReBJCui /tmp/tmp.jsiW8bhVw3 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + proxy=smart-update-haproxy + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h smart-update-pxc-0.smart-update-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5J64WKc0EY +++ mktemp ++ local LAST_ERR=/tmp/tmp.WBJAysymAD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5J64WKc0EY ++ cat /tmp/tmp.WBJAysymAD ++ rm /tmp/tmp.5J64WKc0EY /tmp/tmp.WBJAysymAD ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-rssdb + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:49:03.062167 28762 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967739904757000&timeoutSeconds=507&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data smart-update select-1 no-proxy -P3306 + local cluster=smart-update + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6lWvfaOGvO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.PAzQI9V1FA +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.6lWvfaOGvO +++ cat /tmp/tmp.PAzQI9V1FA +++ rm /tmp/tmp.6lWvfaOGvO /tmp/tmp.PAzQI9V1FA +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.d1pdbalP6b +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ersdr1yDR4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.d1pdbalP6b ++ cat /tmp/tmp.Ersdr1yDR4 ++ rm /tmp/tmp.d1pdbalP6b /tmp/tmp.Ersdr1yDR4 ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rKDI35Dub9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TPDyQgdTkP +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.rKDI35Dub9 +++ cat /tmp/tmp.TPDyQgdTkP +++ rm /tmp/tmp.rKDI35Dub9 /tmp/tmp.TPDyQgdTkP +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.opx40aapJ2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.HOTCN3nIGg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.opx40aapJ2 ++ cat /tmp/tmp.HOTCN3nIGg ++ rm /tmp/tmp.opx40aapJ2 /tmp/tmp.HOTCN3nIGg ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-rssdb + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:49:57.425387 4611 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967794981085000&timeoutSeconds=402&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ESereZZMKp +++ mktemp ++ local LAST_ERR=/tmp/tmp.OD988Ojkxo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ESereZZMKp ++ cat /tmp/tmp.OD988Ojkxo ++ rm /tmp/tmp.ESereZZMKp /tmp/tmp.OD988Ojkxo ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-rssdb ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:50:10.495139 6922 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967809395402000&timeoutSeconds=531&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3H0XtC3JX4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.R0rk0w3whx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3H0XtC3JX4 ++ cat /tmp/tmp.R0rk0w3whx ++ rm /tmp/tmp.3H0XtC3JX4 /tmp/tmp.R0rk0w3whx ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-rssdb + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:50:27.209805 9245 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967824921934000&timeoutSeconds=344&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + kubectl exec smart-update-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my + return 1 + vs_image=latest ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.i7GBN7wQ30 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mMjku0faKr +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.i7GBN7wQ30 +++ cat /tmp/tmp.mMjku0faKr +++ rm /tmp/tmp.i7GBN7wQ30 /tmp/tmp.mMjku0faKr +++ return 0 ++ client_pod=pxc-client-58956d9c99-rssdb ++ wait_pod pxc-client-58956d9c99-rssdb ++ local pod=pxc-client-58956d9c99-rssdb ++ local max_retry=480 ++ local ns= +++ echo pxc-client-58956d9c99-rssdb +++ grep -E '^(pxc|proxysql)$' +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ local container= ++ set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:50:51.270250 12739 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967847744191000&timeoutSeconds=538&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.rIjJL0fbgs ++ mktemp + local LAST_ERR=/tmp/tmp.GHbL1Q9Zm9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rIjJL0fbgs perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.GHbL1Q9Zm9 + rm /tmp/tmp.rIjJL0fbgs /tmp/tmp.GHbL1Q9Zm9 + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pGsFme8Rp0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.PniEYqKouc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pGsFme8Rp0 ++ cat /tmp/tmp.PniEYqKouc ++ rm /tmp/tmp.pGsFme8Rp0 /tmp/tmp.PniEYqKouc ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.b1jXaE2Vm8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.36C8C0UyHL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.b1jXaE2Vm8 ++ cat /tmp/tmp.36C8C0UyHL ++ rm /tmp/tmp.b1jXaE2Vm8 /tmp/tmp.36C8C0UyHL ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.WspKkYMabh ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.jpnBx6OkOr +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.WspKkYMabh +++++ cat /tmp/tmp.jpnBx6OkOr +++++ rm /tmp/tmp.WspKkYMabh /tmp/tmp.jpnBx6OkOr +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7rcB94gYzy +++ mktemp ++ local LAST_ERR=/tmp/tmp.3YfNOHqyti ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7rcB94gYzy ++ cat /tmp/tmp.3YfNOHqyti ++ rm /tmp/tmp.7rcB94gYzy /tmp/tmp.3YfNOHqyti ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hhepToP0x2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.EPVmqJuQ9x ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hhepToP0x2 ++ cat /tmp/tmp.EPVmqJuQ9x ++ rm /tmp/tmp.hhepToP0x2 /tmp/tmp.EPVmqJuQ9x ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-rssdb ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:52:23.896745 27194 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967941053583000&timeoutSeconds=393&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AifvV54JTY +++ mktemp ++ local LAST_ERR=/tmp/tmp.vi1BOOww28 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AifvV54JTY ++ cat /tmp/tmp.vi1BOOww28 ++ rm /tmp/tmp.AifvV54JTY /tmp/tmp.vi1BOOww28 ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-rssdb ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:52:39.328951 29445 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967955926215000&timeoutSeconds=370&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TmBngIQM5n +++ mktemp ++ local LAST_ERR=/tmp/tmp.Rj1V8IaFiZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TmBngIQM5n ++ cat /tmp/tmp.Rj1V8IaFiZ ++ rm /tmp/tmp.TmBngIQM5n /tmp/tmp.Rj1V8IaFiZ ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-58956d9c99-rssdb ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:52:55.652885 31970 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782967972466998000&timeoutSeconds=594&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + kubectl_bin delete -f /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.hra6T28KwD ++ mktemp + local LAST_ERR=/tmp/tmp.6AN85Ig1Xg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hra6T28KwD perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-6497 namespace + cat /tmp/tmp.6AN85Ig1Xg + rm /tmp/tmp.hra6T28KwD /tmp/tmp.6AN85Ig1Xg + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.2RlkOWyYKE ++ mktemp + local LAST_ERR=/tmp/tmp.EMpN0bY0VU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2RlkOWyYKE persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-6497 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-6497 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-6497 namespace + cat /tmp/tmp.EMpN0bY0VU + rm /tmp/tmp.2RlkOWyYKE /tmp/tmp.EMpN0bY0VU + return 0 + desc 'PXC cluster update with explicitly specified image inside version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with explicitly specified image inside version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.rvLpjHzc12 ++ mktemp + local LAST_ERR=/tmp/tmp.UlaXW2b7gY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rvLpjHzc12 secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.UlaXW2b7gY + rm /tmp/tmp.rvLpjHzc12 /tmp/tmp.UlaXW2b7gY + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-6497~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.tIPEchuiuY ++ mktemp + local LAST_ERR=/tmp/tmp.4TzQDCi1Nn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tIPEchuiuY deployment.apps/pxc-client unchanged + cat /tmp/tmp.4TzQDCi1Nn + rm /tmp/tmp.tIPEchuiuY /tmp/tmp.4TzQDCi1Nn + return 0 + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + apply_config /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local config_file=/tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local pvc_name= + '[' -z '' ']' + cat_config /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml '' + kubectl_bin apply -f - + local input_file=/tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local pvc_name= + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + cat /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + local LAST_OUT=/tmp/tmp.GzMByquRga + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-6497~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + local LAST_ERR=/tmp/tmp.I1dbU19aFR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GzMByquRga perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.I1dbU19aFR + rm /tmp/tmp.GzMByquRga /tmp/tmp.I1dbU19aFR + return 0 + desc 'waiting pods to be running for smart-update' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for smart-update ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Qpru54Wcth ++++ mktemp +++ local LAST_ERR=/tmp/tmp.U7bUHkmfVA +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Qpru54Wcth +++ cat /tmp/tmp.U7bUHkmfVA +++ rm /tmp/tmp.Qpru54Wcth /tmp/tmp.U7bUHkmfVA +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy ++ get_proxy_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qcCSEpj4hJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wm3NwD6nNl +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.qcCSEpj4hJ +++ cat /tmp/tmp.wm3NwD6nNl +++ rm /tmp/tmp.qcCSEpj4hJ /tmp/tmp.wm3NwD6nNl +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qxerPwz4oE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1KsC7FZ6mc +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.qxerPwz4oE +++ cat /tmp/tmp.1KsC7FZ6mc +++ rm /tmp/tmp.qxerPwz4oE /tmp/tmp.1KsC7FZ6mc +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z smart-update-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for smart-update-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for smart-update-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + wait_for_running smart-update-haproxy 2 + local name=smart-update-haproxy + local size=2 + local last_pod + local max_retry=480 + [[ -z smart-update-haproxy ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for smart-update-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for smart-update-haproxy ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met E0702 04:58:29.937006 17486 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-0&resourceVersion=1782968306715045000&timeoutSeconds=305&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-1 480 + local pod=smart-update-haproxy-1 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-1 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/smart-update-haproxy-1 condition met E0702 04:58:45.253650 19600 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-1&resourceVersion=1782968321715113000&timeoutSeconds=530&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-1 to become Ready.Ok + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VRCbuurBic +++ mktemp ++ local LAST_ERR=/tmp/tmp.YOJlNFrUtb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VRCbuurBic ++ cat /tmp/tmp.YOJlNFrUtb ++ rm /tmp/tmp.VRCbuurBic /tmp/tmp.YOJlNFrUtb ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CO5Ul2WyQH +++ mktemp ++ local LAST_ERR=/tmp/tmp.x6M6RmhOfz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CO5Ul2WyQH ++ cat /tmp/tmp.x6M6RmhOfz ++ rm /tmp/tmp.CO5Ul2WyQH /tmp/tmp.x6M6RmhOfz ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.7qR3ji3pBK ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.zycZOe0jGt +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.7qR3ji3pBK +++++ cat /tmp/tmp.zycZOe0jGt +++++ rm /tmp/tmp.7qR3ji3pBK /tmp/tmp.zycZOe0jGt +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IXIxeTKlta +++ mktemp ++ local LAST_ERR=/tmp/tmp.E5xAWvP4lw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IXIxeTKlta ++ cat /tmp/tmp.E5xAWvP4lw ++ rm /tmp/tmp.IXIxeTKlta /tmp/tmp.E5xAWvP4lw ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UF7HnNnF8Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZZbx7vg7ug ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UF7HnNnF8Z ++ cat /tmp/tmp.ZZbx7vg7ug ++ rm /tmp/tmp.UF7HnNnF8Z /tmp/tmp.ZZbx7vg7ug ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d5gCjokdJf +++ mktemp ++ local LAST_ERR=/tmp/tmp.RGgMFPPBza ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.d5gCjokdJf ++ cat /tmp/tmp.RGgMFPPBza ++ rm /tmp/tmp.d5gCjokdJf /tmp/tmp.RGgMFPPBza ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-rssdb ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:59:28.690199 26110 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782968365593039000&timeoutSeconds=300&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data smart-update 100500 no-proxy -P3306 + local cluster=smart-update + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.w1Bqtp9ncg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LATrwxUzSB +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.w1Bqtp9ncg +++ cat /tmp/tmp.LATrwxUzSB +++ rm /tmp/tmp.w1Bqtp9ncg /tmp/tmp.LATrwxUzSB +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SHMIXgFwPr +++ mktemp ++ local LAST_ERR=/tmp/tmp.rJ1p0bXbdh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SHMIXgFwPr ++ cat /tmp/tmp.rJ1p0bXbdh ++ rm /tmp/tmp.SHMIXgFwPr /tmp/tmp.rJ1p0bXbdh ++ return 0 + root_pass=root_password ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JbLiLnHAhJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iHHf1VhloQ +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.JbLiLnHAhJ +++ cat /tmp/tmp.iHHf1VhloQ +++ rm /tmp/tmp.JbLiLnHAhJ /tmp/tmp.iHHf1VhloQ +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + proxy=smart-update-haproxy + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h smart-update-pxc-0.smart-update-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FNePzDyfNk +++ mktemp ++ local LAST_ERR=/tmp/tmp.GKCX6F7vyY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FNePzDyfNk ++ cat /tmp/tmp.GKCX6F7vyY ++ rm /tmp/tmp.FNePzDyfNk /tmp/tmp.GKCX6F7vyY ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-rssdb ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 04:59:51.368538 29632 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782968390377968000&timeoutSeconds=461&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data smart-update select-1 no-proxy -P3306 + local cluster=smart-update + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vIJUopwTwx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6dUtGmy3eu +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.vIJUopwTwx +++ cat /tmp/tmp.6dUtGmy3eu +++ rm /tmp/tmp.vIJUopwTwx /tmp/tmp.6dUtGmy3eu +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NwmxATgYAl +++ mktemp ++ local LAST_ERR=/tmp/tmp.nozIjJ3rHu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NwmxATgYAl ++ cat /tmp/tmp.nozIjJ3rHu ++ rm /tmp/tmp.NwmxATgYAl /tmp/tmp.nozIjJ3rHu ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hJzkvmFzUV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gM4NaLstLh +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.hJzkvmFzUV +++ cat /tmp/tmp.gM4NaLstLh +++ rm /tmp/tmp.hJzkvmFzUV /tmp/tmp.gM4NaLstLh +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MpMa4qsFpA +++ mktemp ++ local LAST_ERR=/tmp/tmp.iwRrwYOAyO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MpMa4qsFpA ++ cat /tmp/tmp.iwRrwYOAyO ++ rm /tmp/tmp.MpMa4qsFpA /tmp/tmp.iwRrwYOAyO ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-rssdb ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 05:00:34.506309 3150 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782968432712054000&timeoutSeconds=450&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VMjR9HYQdj +++ mktemp ++ local LAST_ERR=/tmp/tmp.0cX4OmVud3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VMjR9HYQdj ++ cat /tmp/tmp.0cX4OmVud3 ++ rm /tmp/tmp.VMjR9HYQdj /tmp/tmp.0cX4OmVud3 ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-58956d9c99-rssdb + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 05:00:46.481806 4901 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782968444313273000&timeoutSeconds=332&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Yh9rWhtJMc +++ mktemp ++ local LAST_ERR=/tmp/tmp.cLEbxotBgu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Yh9rWhtJMc ++ cat /tmp/tmp.cLEbxotBgu ++ rm /tmp/tmp.Yh9rWhtJMc /tmp/tmp.cLEbxotBgu ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-rssdb ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 05:00:57.189262 6495 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782968455465995000&timeoutSeconds=303&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + kubectl exec smart-update-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my + return 1 ++ head -n1 ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ grep :8.4 ++ tail -n2 ++ sort -V +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++++ grep -c percona-xtradb-cluster-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++++ mktemp +++ head -1 +++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ local LAST_OUT=/tmp/tmp.opPfNoX3Mb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.W4cOXVldDr +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.opPfNoX3Mb +++ cat /tmp/tmp.W4cOXVldDr +++ rm /tmp/tmp.opPfNoX3Mb /tmp/tmp.W4cOXVldDr +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-b7b8b9959-x4wnq -n pxc-operator -- curl -s http://version-service.smart-update2-6497.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.tt0COaQm6K +++ mktemp ++ local LAST_ERR=/tmp/tmp.ar5ZqwuxvF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-b7b8b9959-x4wnq -n pxc-operator -- curl -s http://version-service.smart-update2-6497.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tt0COaQm6K ++ cat /tmp/tmp.ar5ZqwuxvF Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.tt0COaQm6K /tmp/tmp.ar5ZqwuxvF ++ return 0 + vs_image=percona/percona-xtradb-cluster:8.4.5-5.1 ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.h5rew3fbm2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WxrdDM5WAE +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.h5rew3fbm2 +++ cat /tmp/tmp.WxrdDM5WAE +++ rm /tmp/tmp.h5rew3fbm2 /tmp/tmp.WxrdDM5WAE +++ return 0 ++ client_pod=pxc-client-58956d9c99-rssdb ++ wait_pod pxc-client-58956d9c99-rssdb ++ local pod=pxc-client-58956d9c99-rssdb ++ local max_retry=480 ++ local ns= +++ grep -E '^(pxc|proxysql)$' +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ echo pxc-client-58956d9c99-rssdb ++ local container= ++ set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 05:01:13.140024 8885 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782968471616142000&timeoutSeconds=320&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.4.5-5.1","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.oBcx5OTTYA ++ mktemp + local LAST_ERR=/tmp/tmp.YqmhVaOW26 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.4.5-5.1","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oBcx5OTTYA perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.YqmhVaOW26 + rm /tmp/tmp.oBcx5OTTYA /tmp/tmp.YqmhVaOW26 + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.4.5-5.1 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.4.5-5.1 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AJ4H7cKLZx +++ mktemp ++ local LAST_ERR=/tmp/tmp.zZrGcAxcwc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AJ4H7cKLZx ++ cat /tmp/tmp.zZrGcAxcwc ++ rm /tmp/tmp.AJ4H7cKLZx /tmp/tmp.zZrGcAxcwc ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nG4J8a3zui +++ mktemp ++ local LAST_ERR=/tmp/tmp.C4n55yc1PN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nG4J8a3zui ++ cat /tmp/tmp.C4n55yc1PN ++ rm /tmp/tmp.nG4J8a3zui /tmp/tmp.C4n55yc1PN ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.gZIyFbXLzg ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.pupj0PKlHb +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.gZIyFbXLzg +++++ cat /tmp/tmp.pupj0PKlHb +++++ rm /tmp/tmp.gZIyFbXLzg /tmp/tmp.pupj0PKlHb +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GKr9jjU334 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ic2H4dG6Vg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GKr9jjU334 ++ cat /tmp/tmp.ic2H4dG6Vg ++ rm /tmp/tmp.GKr9jjU334 /tmp/tmp.ic2H4dG6Vg ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cTiygj24tk +++ mktemp ++ local LAST_ERR=/tmp/tmp.KXHkvCwyYp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cTiygj24tk ++ cat /tmp/tmp.KXHkvCwyYp ++ rm /tmp/tmp.cTiygj24tk /tmp/tmp.KXHkvCwyYp ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-58956d9c99-rssdb + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 05:02:34.819812 18811 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782968552485409000&timeoutSeconds=522&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sIvLUDLV1c +++ mktemp ++ local LAST_ERR=/tmp/tmp.PZRo9oGHgM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sIvLUDLV1c ++ cat /tmp/tmp.PZRo9oGHgM ++ rm /tmp/tmp.sIvLUDLV1c /tmp/tmp.PZRo9oGHgM ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-rssdb ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 05:02:43.387712 20190 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782968562555624000&timeoutSeconds=391&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.a12qbByBSA +++ mktemp ++ local LAST_ERR=/tmp/tmp.GB2xV0BHBJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.a12qbByBSA ++ cat /tmp/tmp.GB2xV0BHBJ ++ rm /tmp/tmp.a12qbByBSA /tmp/tmp.GB2xV0BHBJ ++ return 0 + client_pod=pxc-client-58956d9c99-rssdb + wait_pod pxc-client-58956d9c99-rssdb + local pod=pxc-client-58956d9c99-rssdb + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-rssdb ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-rssdb condition met E0702 05:02:50.861764 21164 reflector.go:227] "Failed to watch" err="Get \"https://34.9.252.246/api/v1/namespaces/smart-update2-6497/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-58956d9c99-rssdb&resourceVersion=1782968569472313000&timeoutSeconds=500&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-58956d9c99-rssdb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.7fN8FXUd19/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.7fN8FXUd19/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7fN8FXUd19/select-1.sql + kubectl_bin delete -f /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.NZeOXmTUzq ++ mktemp + local LAST_ERR=/tmp/tmp.H8WpiZKUEH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.7fN8FXUd19/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NZeOXmTUzq perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-6497 namespace + cat /tmp/tmp.H8WpiZKUEH + rm /tmp/tmp.NZeOXmTUzq /tmp/tmp.H8WpiZKUEH + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.iQvEGfWzeJ ++ mktemp + local LAST_ERR=/tmp/tmp.W6AurSf421 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iQvEGfWzeJ persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-6497 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-6497 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-6497 namespace + cat /tmp/tmp.W6AurSf421 + rm /tmp/tmp.iQvEGfWzeJ /tmp/tmp.W6AurSf421 + return 0 + desc cleanup + set +o xtrace ----------------------------------------------------------------------------------- cleanup ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.8PvNzqGth0 ++ mktemp + local LAST_ERR=/tmp/tmp.YwRbikaHFP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update2/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8PvNzqGth0 deployment.apps "version-service" deleted from smart-update2-6497 namespace service "version-service" deleted from smart-update2-6497 namespace + cat /tmp/tmp.YwRbikaHFP + rm /tmp/tmp.8PvNzqGth0 /tmp/tmp.YwRbikaHFP + return 0 + destroy smart-update2-6497 + local namespace=smart-update2-6497 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v 'the object has been modified' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + sort -u + tee /tmp/tmp.7fN8FXUd19/operator.log ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator + grep -v 'get backup status: Job.batch' + grep -v level=info ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.hxdsqHkOeP +++ mktemp ++ local LAST_ERR=/tmp/tmp.3LXiZrS0HP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hxdsqHkOeP ++ cat /tmp/tmp.3LXiZrS0HP ++ rm /tmp/tmp.hxdsqHkOeP /tmp/tmp.3LXiZrS0HP ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-b7b8b9959-x4wnq ++ mktemp + local LAST_OUT=/tmp/tmp.3VYJIvzFdm ++ mktemp + local LAST_ERR=/tmp/tmp.0qjux1z2wA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-b7b8b9959-x4wnq + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3VYJIvzFdm + cat /tmp/tmp.0qjux1z2wA + rm /tmp/tmp.3VYJIvzFdm /tmp/tmp.0qjux1z2wA + return 0 2026-07-02T04:32:32.132Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-07-02T04:32:32.132Z INFO setup Manager starting up {"gitCommit": "70c04858efdd60ae80fd1cf4dcb3edc388dbd027", "gitBranch": "PR-2538-70c04858", "buildTime": "2026-07-02T01:14:54Z", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} 2026-07-02T04:32:32.132Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.33.12-gke.1059000"} 2026-07-02T04:32:32.136Z INFO setup Registering Components. 2026-07-02T04:32:32.157Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-07-02T04:32:32.157Z INFO controller-runtime.metrics Starting metrics server 2026-07-02T04:32:32.157Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-07-02T04:32:32.157Z INFO controller-runtime.webhook Starting webhook server 2026-07-02T04:32:32.157Z INFO setup Starting the Cmd. 2026-07-02T04:32:32.157Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-07-02T04:32:32.158Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-07-02T04:32:32.158Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-07-02T04:32:32.158Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-07-02T04:32:32.359Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-07-02T04:32:54.610Z DEBUG events percona-xtradb-cluster-operator-b7b8b9959-x4wnq_ff929624-0d00-4e8b-8161-5616f00a413d became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"1ea11a38-6951-49c2-af12-323c7a0a091f","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1782966774602367009"}, "reason": "LeaderElection"} 2026-07-02T04:32:54.610Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-07-02T04:32:54.610Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-07-02T04:32:54.610Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-07-02T04:32:54.610Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-07-02T04:32:54.611Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-07-02T04:32:54.712Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-07-02T04:32:54.712Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-07-02T04:32:54.811Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-07-02T04:32:54.811Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-07-02T04:32:54.811Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-07-02T04:32:54.811Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-07-02T04:32:59.844Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"minimal-cluster","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "minimal-cluster", "reconcileID": "ec80cef0-0a4a-49bd-9fcb-f5839003f0b9", "error": "PerconaXtraDBCluster.pxc.percona.com \"minimal-cluster\" not found"} 2026-07-02T04:33:46.414Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "d6f93f4a-3c97-46e3-adf7-dfaaefe1cee5", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:33:46.430Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "d6f93f4a-3c97-46e3-adf7-dfaaefe1cee5", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:33:46.987Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "d6f93f4a-3c97-46e3-adf7-dfaaefe1cee5", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:33:47.112Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "ddeecd4e-6263-4856-a06e-2cbee08967f6", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:33:47.155Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "ddeecd4e-6263-4856-a06e-2cbee08967f6", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:33:47.208Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "ddeecd4e-6263-4856-a06e-2cbee08967f6", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:33:47.263Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "ddeecd4e-6263-4856-a06e-2cbee08967f6", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:33:47.538Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "ddeecd4e-6263-4856-a06e-2cbee08967f6", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:33:47.934Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "ddeecd4e-6263-4856-a06e-2cbee08967f6", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:33:48.900Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "05aa3f03-7603-45a5-81e3-2f4c2008e4e9", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:33:48.921Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "05aa3f03-7603-45a5-81e3-2f4c2008e4e9", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:35:05.752Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "eed4df30-22f5-43c4-89a5-527038638a6b", "user": "operator"} 2026-07-02T04:35:05.786Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "eed4df30-22f5-43c4-89a5-527038638a6b", "user": "monitor"} 2026-07-02T04:35:05.834Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "eed4df30-22f5-43c4-89a5-527038638a6b"} 2026-07-02T04:35:05.863Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "eed4df30-22f5-43c4-89a5-527038638a6b"} 2026-07-02T04:35:05.896Z INFO monitor user audit admin privilege granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "eed4df30-22f5-43c4-89a5-527038638a6b"} 2026-07-02T04:35:05.931Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "eed4df30-22f5-43c4-89a5-527038638a6b", "user": "xtrabackup"} 2026-07-02T04:35:05.968Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "eed4df30-22f5-43c4-89a5-527038638a6b"} 2026-07-02T04:35:05.997Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "eed4df30-22f5-43c4-89a5-527038638a6b", "user": "replication"} 2026-07-02T04:37:48.434Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "9f472113-32a3-40cf-a67d-53c0fdb0e576", "user": "root"} 2026-07-02T04:37:49.502Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "9f472113-32a3-40cf-a67d-53c0fdb0e576", "new version": "8.4.5-5.1"} 2026-07-02T04:40:45.439Z INFO add new job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "2b424830-a428-49ba-9651-b2920e94ff65", "name": "ensure-version/smart-update2-6497/smart-update", "schedule": "* * * * *"} 2026-07-02T04:41:45.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "2b424830-a428-49ba-9651-b2920e94ff65", "endpoint": "http://version-service.smart-update2-6497.svc.cluster.local:11000"} 2026-07-02T04:41:45.061Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "2b424830-a428-49ba-9651-b2920e94ff65", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=2460e011-bf48-42dd-8de8-a92095626d18&databaseVersion=8.4.5-5.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=2460e011-bf48-42dd-8de8-a92095626d18&databaseVersion=8.4.5-5.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:42:45.001Z INFO cluster is not ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "2b424830-a428-49ba-9651-b2920e94ff65"} 2026-07-02T04:43:20.525Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "70deb743-5e45-4386-9bc5-7e5af2cc9a1a", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2026-07-02T04:43:45.001Z INFO cluster is not found, deleting the job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "2b424830-a428-49ba-9651-b2920e94ff65", "name": "ensure-version/smart-update2-6497/smart-update", "cluster": "smart-update", "namespace": "smart-update2-6497"} 2026-07-02T04:43:46.771Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "9e5aeb1c-c81e-486b-8ca2-c2c74012c191", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:43:46.787Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "9e5aeb1c-c81e-486b-8ca2-c2c74012c191", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:43:47.341Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "9e5aeb1c-c81e-486b-8ca2-c2c74012c191", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:43:47.748Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "bc2b0000-40dc-48b7-a4be-13e846f2ff8f", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:43:47.821Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "bc2b0000-40dc-48b7-a4be-13e846f2ff8f", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:43:47.916Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "bc2b0000-40dc-48b7-a4be-13e846f2ff8f", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:43:47.984Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "bc2b0000-40dc-48b7-a4be-13e846f2ff8f", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:43:48.029Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "bc2b0000-40dc-48b7-a4be-13e846f2ff8f", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:43:48.245Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "bc2b0000-40dc-48b7-a4be-13e846f2ff8f", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:43:49.064Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "9c3a62eb-4f9c-428e-9fae-51ae77d2694a", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:43:49.082Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "9c3a62eb-4f9c-428e-9fae-51ae77d2694a", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:47:37.765Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "ee4b8e97-bb0a-49d3-8d68-b280ceb3f469", "err": "failed to ensure cluster readonly status: connect to pod smart-update-pxc-2: dial tcp: lookup smart-update-pxc-2.smart-update-pxc.smart-update2-6497 on 34.118.224.10:53: no such host"} 2026-07-02T04:47:39.577Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "cbd5b478-a0c6-4851-9d5e-f6ad0d02b40f", "new version": "8.4.5-5.1"} 2026-07-02T04:51:06.467Z INFO add new job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "e609ee2b-24a4-40bc-aad2-4d34a94c7e2d", "name": "ensure-version/smart-update2-6497/smart-update", "schedule": "* * * * *"} 2026-07-02T04:52:06.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "e609ee2b-24a4-40bc-aad2-4d34a94c7e2d", "endpoint": "http://version-service.smart-update2-6497.svc.cluster.local:11000"} 2026-07-02T04:52:06.066Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "e609ee2b-24a4-40bc-aad2-4d34a94c7e2d", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=7f777d32-72d0-49af-a789-6438eaca7ddc&databaseVersion=8.4.5-5.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=7f777d32-72d0-49af-a789-6438eaca7ddc&databaseVersion=8.4.5-5.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:53:07.078Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "e609ee2b-24a4-40bc-aad2-4d34a94c7e2d", "endpoint": "http://version-service.smart-update2-6497.svc.cluster.local:11000"} 2026-07-02T04:53:07.948Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "e609ee2b-24a4-40bc-aad2-4d34a94c7e2d", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=7f777d32-72d0-49af-a789-6438eaca7ddc&databaseVersion=8.4.5-5.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=7f777d32-72d0-49af-a789-6438eaca7ddc&databaseVersion=8.4.5-5.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:53:49.458Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "31436692-b51c-4f7c-9db9-d156297a594a", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2026-07-02T04:54:06.001Z INFO cluster is not found, deleting the job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "e609ee2b-24a4-40bc-aad2-4d34a94c7e2d", "name": "ensure-version/smart-update2-6497/smart-update", "cluster": "smart-update", "namespace": "smart-update2-6497"} 2026-07-02T04:54:15.544Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "3f105f58-8016-4dad-8d41-4f25367e51a5", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:54:15.561Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "3f105f58-8016-4dad-8d41-4f25367e51a5", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:54:16.127Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "3f105f58-8016-4dad-8d41-4f25367e51a5", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:54:16.242Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "d56c36ee-fa78-4a64-88c2-d14feaacecdb", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:54:16.291Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "d56c36ee-fa78-4a64-88c2-d14feaacecdb", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:54:16.346Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "d56c36ee-fa78-4a64-88c2-d14feaacecdb", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:54:16.426Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "d56c36ee-fa78-4a64-88c2-d14feaacecdb", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:54:16.483Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "d56c36ee-fa78-4a64-88c2-d14feaacecdb", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:54:16.660Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "d56c36ee-fa78-4a64-88c2-d14feaacecdb", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:54:17.624Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "6d1b173d-44e6-41d3-aabb-b051059826c4", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:54:17.672Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "6d1b173d-44e6-41d3-aabb-b051059826c4", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:55:41.295Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "14fd5e9b-457f-4166-940f-43276272c6f8", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.229.45:3306: connect: connection refused"} 2026-07-02T04:58:22.385Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "bb2c41bc-cb97-4536-ad7a-066e8b1ab524", "new version": "8.4.5-5.1"} 2026-07-02T05:01:20.346Z INFO add new job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "a4663491-9ac7-4e97-8914-d3d39dc368bf", "name": "ensure-version/smart-update2-6497/smart-update", "schedule": "* * * * *"} 2026-07-02T05:02:20.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "a4663491-9ac7-4e97-8914-d3d39dc368bf", "endpoint": "http://version-service.smart-update2-6497.svc.cluster.local:11000"} 2026-07-02T05:02:20.081Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "a4663491-9ac7-4e97-8914-d3d39dc368bf", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.4.5-5.1?clusterWideEnabled=true&customResourceUid=871570a3-460b-4b71-bc1f-0bd82b19d413&databaseVersion=8.4.5-5.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.4.5-5.1?clusterWideEnabled=true&customResourceUid=871570a3-460b-4b71-bc1f-0bd82b19d413&databaseVersion=8.4.5-5.1&kubeVersion=v1.33.12-gke.1059000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T05:03:20.001Z INFO cluster is not ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "a4663491-9ac7-4e97-8914-d3d39dc368bf"} 2026-07-02T05:03:36.276Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-6497"}, "namespace": "smart-update2-6497", "name": "smart-update", "reconcileID": "11571dc4-9663-4d04-a19e-10a29f776390", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1 github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1 github.com/robfig/cron/v3.(*Cron).startJob.func1 github.com/robfig/cron/v3.FuncJob.Run /go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136 /go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:272 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:509 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:215 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 + local backup_names ++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + backup_names= + [[ -n '' ]] + local pxc_names ++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + pxc_names= + [[ -n '' ]] + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.YTnu7HTu1Z ++ mktemp + local LAST_ERR=/tmp/tmp.95nZrS9hAm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YTnu7HTu1Z validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.95nZrS9hAm + rm /tmp/tmp.YTnu7HTu1Z /tmp/tmp.95nZrS9hAm + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml namespace "cert-manager" deleted customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted serviceaccount "cert-manager-cainjector" deleted from cert-manager namespace serviceaccount "cert-manager" deleted from cert-manager namespace serviceaccount "cert-manager-webhook" deleted from cert-manager namespace clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace role.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace service "cert-manager-cainjector" deleted from cert-manager namespace service "cert-manager" deleted from cert-manager namespace service "cert-manager-webhook" deleted from cert-manager namespace deployment.apps "cert-manager-cainjector" deleted from cert-manager namespace deployment.apps "cert-manager" deleted from cert-manager namespace deployment.apps "cert-manager-webhook" deleted from cert-manager namespace mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace smart-update2-6497 + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ date +%s ++ mktemp + local LAST_OUT=/tmp/tmp.Iy8QfZU33m + local deadline=1782968786 ++ mktemp + local LAST_OUT=/tmp/tmp.cBIPDkU4Up ++ mktemp ++ kubectl_bin get namespace smart-update2-6497 --ignore-not-found -o name + local LAST_ERR=/tmp/tmp.sbrra5J2WK + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.sX6amKdOxk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace smart-update2-6497 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + [[ -z namespace/smart-update2-6497 ]] ++ date +%s + [[ 1782968668 -ge 1782968786 ]] + sleep 5 ++ kubectl_bin get namespace smart-update2-6497 --ignore-not-found -o name + [[ -z namespace/smart-update2-6497 ]] ++ date +%s + [[ 1782968674 -ge 1782968786 ]] + sleep 5 ++ kubectl_bin get namespace smart-update2-6497 --ignore-not-found -o name + [[ -z namespace/smart-update2-6497 ]] ++ date +%s + [[ 1782968680 -ge 1782968786 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cBIPDkU4Up namespace "pxc-operator" force deleted + cat /tmp/tmp.sX6amKdOxk Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.cBIPDkU4Up /tmp/tmp.sX6amKdOxk + return 0 ++ kubectl_bin get namespace smart-update2-6497 --ignore-not-found -o name + [[ -z namespace/smart-update2-6497 ]] ++ date +%s + [[ 1782968687 -ge 1782968786 ]] + sleep 5 ++ kubectl_bin get namespace smart-update2-6497 --ignore-not-found -o name + [[ -z namespace/smart-update2-6497 ]] ++ date +%s + [[ 1782968693 -ge 1782968786 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Iy8QfZU33m namespace "smart-update2-6497" force deleted + cat /tmp/tmp.sbrra5J2WK Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.Iy8QfZU33m /tmp/tmp.sbrra5J2WK + return 0 ++ kubectl_bin get namespace smart-update2-6497 --ignore-not-found -o name + [[ -z '' ]] ++ date +%s + deadline=1782968819 ++ awk '$2 != ""' ++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' + [[ -z '' ]] + rm -rf /tmp/tmp.7fN8FXUd19 + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------