Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/logs/smart-update1-8-0.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + API=pxc.percona.com/v9-9-9 + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + CLUSTER=smart-update + CLUSTER_SIZE=3 + PROXY_SIZE=2 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0 ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' + PXC_VER=8.0 + TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + VS_URL=http://version-service + VS_PORT=11000 + VS_ENDPOINT=http://version-service:11000 + main + create_infra smart-update1-19768 + local ns=smart-update1-19768 + '[' -n pxc-operator ']' + read -r pns name + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + [[ -z smart-update1-26771 ]] + [[ -z smart-update ]] + kubectl_bin patch pxc -n smart-update1-26771 smart-update --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.i8p1SVYAt9 ++ mktemp + local LAST_ERR=/tmp/tmp.DJpWBP57j1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc -n smart-update1-26771 smart-update --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.i8p1SVYAt9 perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.DJpWBP57j1 + rm /tmp/tmp.i8p1SVYAt9 /tmp/tmp.DJpWBP57j1 + return 0 + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update1-26771 namespace + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh.org ++ awk '{print $1}' ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + kubectl_bin get ns + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.tXg9i8WVNG + local LAST_OUT=/tmp/tmp.SbZpajs2hB ++ mktemp + local LAST_ERR=/tmp/tmp.X040Oo4DqZ + local exit_status=0 ++ mktemp + local LAST_ERR=/tmp/tmp.13Cbg0Tdmi + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SbZpajs2hB + cat /tmp/tmp.X040Oo4DqZ + rm /tmp/tmp.SbZpajs2hB /tmp/tmp.X040Oo4DqZ + return 0 namespace "cert-manager" deleted namespace "smart-update1-26771" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tXg9i8WVNG namespace "pxc-operator" deleted + cat /tmp/tmp.13Cbg0Tdmi + rm /tmp/tmp.tXg9i8WVNG /tmp/tmp.13Cbg0Tdmi + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.s7p2RT41fX ++ mktemp + local LAST_ERR=/tmp/tmp.1EDlLJft4p + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.s7p2RT41fX namespace/pxc-operator created + cat /tmp/tmp.1EDlLJft4p + rm /tmp/tmp.s7p2RT41fX /tmp/tmp.1EDlLJft4p + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.pBSvewqyx3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.c2tiFg1rRa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pBSvewqyx3 ++ cat /tmp/tmp.c2tiFg1rRa ++ rm /tmp/tmp.pBSvewqyx3 /tmp/tmp.c2tiFg1rRa ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster16 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ZQPwPcMVJr ++ mktemp + local LAST_ERR=/tmp/tmp.zIeClSn73M + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster16 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZQPwPcMVJr Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster16" modified. + cat /tmp/tmp.zIeClSn73M + rm /tmp/tmp.ZQPwPcMVJr /tmp/tmp.zIeClSn73M + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.m7X0B7sGFf ++ mktemp + local LAST_ERR=/tmp/tmp.McYu9O8IU2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.m7X0B7sGFf customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.McYu9O8IU2 + rm /tmp/tmp.m7X0B7sGFf /tmp/tmp.McYu9O8IU2 + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.Y90xZJcKn4 ++ mktemp + local LAST_ERR=/tmp/tmp.W7MoAjVurE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Y90xZJcKn4 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.W7MoAjVurE E0702 04:17:50.746352 27947 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusters.pxc.percona.com&resourceVersion=1782965868944895007&timeoutSeconds=547&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.Y90xZJcKn4 /tmp/tmp.W7MoAjVurE + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.XZcOfr44my ++ mktemp + local LAST_ERR=/tmp/tmp.mhAFntb8ve + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XZcOfr44my customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.mhAFntb8ve E0702 04:17:52.145276 28043 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterbackups.pxc.percona.com&resourceVersion=1782965868944895007&timeoutSeconds=559&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.XZcOfr44my /tmp/tmp.mhAFntb8ve + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.wQPmV6FSjf ++ mktemp + local LAST_ERR=/tmp/tmp.AxyBhdssKi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wQPmV6FSjf customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.AxyBhdssKi E0702 04:17:53.604944 28170 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterrestores.pxc.percona.com&resourceVersion=1782965868944895007&timeoutSeconds=316&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.wQPmV6FSjf /tmp/tmp.AxyBhdssKi + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.KPKcVklFYG ++ mktemp + local LAST_ERR=/tmp/tmp.BXsJYQVYdR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KPKcVklFYG clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.BXsJYQVYdR + rm /tmp/tmp.KPKcVklFYG /tmp/tmp.BXsJYQVYdR + return 0 + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + kubectl_bin apply -f - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/deploy/cw-operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.UOkRVskQBG ++ mktemp + local LAST_ERR=/tmp/tmp.pWi6SvEgjf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UOkRVskQBG deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.pWi6SvEgjf + rm /tmp/tmp.UOkRVskQBG /tmp/tmp.pWi6SvEgjf + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.tLUFWuNHFG ++ mktemp + local LAST_ERR=/tmp/tmp.KD0EKgjp71 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tLUFWuNHFG pod/percona-xtradb-cluster-operator-59848dbdcc-crj4g condition met + cat /tmp/tmp.KD0EKgjp71 E0702 04:18:11.092667 30459 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-59848dbdcc-crj4g&resourceVersion=1782965890731699000&timeoutSeconds=415&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.tLUFWuNHFG /tmp/tmp.KD0EKgjp71 + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.3diwgFzDhu +++ mktemp ++ local LAST_ERR=/tmp/tmp.z3KREJbzu4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3diwgFzDhu ++ cat /tmp/tmp.z3KREJbzu4 ++ rm /tmp/tmp.3diwgFzDhu /tmp/tmp.z3KREJbzu4 ++ return 0 + wait_pod percona-xtradb-cluster-operator-59848dbdcc-crj4g 480 pxc-operator + local pod=percona-xtradb-cluster-operator-59848dbdcc-crj4g + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-59848dbdcc-crj4g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-59848dbdcc-crj4g condition met E0702 04:18:14.556225 30948 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-59848dbdcc-crj4g&resourceVersion=1782965892861905000&timeoutSeconds=479&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-59848dbdcc-crj4g to become Ready.Ok + sleep 3 + create_namespace smart-update1-19768 + local namespace=smart-update1-19768 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces smart-update1-19768' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces smart-update1-19768 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace smart-update1-19768 ++ mktemp ++ mktemp + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + local LAST_OUT=/tmp/tmp.QuxDa1GlMm + local LAST_OUT=/tmp/tmp.HyLCGMcXj0 ++ mktemp + local LAST_ERR=/tmp/tmp.Q5kbLcimBu + local exit_status=0 ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.UJhHw0huAP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-19768 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HyLCGMcXj0 + cat /tmp/tmp.UJhHw0huAP + rm /tmp/tmp.HyLCGMcXj0 /tmp/tmp.UJhHw0huAP + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-19768 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-19768 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.QuxDa1GlMm + cat /tmp/tmp.Q5kbLcimBu Error from server (NotFound): namespaces "smart-update1-19768" not found + rm /tmp/tmp.QuxDa1GlMm /tmp/tmp.Q5kbLcimBu + return 1 + : + wait_for_delete namespace/smart-update1-19768 + local res=namespace/smart-update1-19768 + echo -n 'waiting for namespace/smart-update1-19768 to be deleted' waiting for namespace/smart-update1-19768 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "smart-update1-19768" not found + desc 'create namespace smart-update1-19768' + set +o xtrace ----------------------------------------------------------------------------------- create namespace smart-update1-19768 ----------------------------------------------------------------------------------- + kubectl_bin create namespace smart-update1-19768 ++ mktemp + local LAST_OUT=/tmp/tmp.e1v4gr9GYT ++ mktemp + local LAST_ERR=/tmp/tmp.d4C8dHc1K9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace smart-update1-19768 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.e1v4gr9GYT namespace/smart-update1-19768 created + cat /tmp/tmp.d4C8dHc1K9 + rm /tmp/tmp.e1v4gr9GYT /tmp/tmp.d4C8dHc1K9 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.ma35Ht0oDU +++ mktemp ++ local LAST_ERR=/tmp/tmp.trKgvGrj4v ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ma35Ht0oDU ++ cat /tmp/tmp.trKgvGrj4v ++ rm /tmp/tmp.ma35Ht0oDU /tmp/tmp.trKgvGrj4v ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster16 --namespace=smart-update1-19768 ++ mktemp + local LAST_OUT=/tmp/tmp.NyvUYZf4Ws ++ mktemp + local LAST_ERR=/tmp/tmp.5KrRgOncTY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster16 --namespace=smart-update1-19768 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NyvUYZf4Ws Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2538-70c04858-1-cluster16" modified. + cat /tmp/tmp.5KrRgOncTY + rm /tmp/tmp.NyvUYZf4Ws /tmp/tmp.5KrRgOncTY + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.73Zd0hd3Yq ++ mktemp + local LAST_ERR=/tmp/tmp.leAFVmubK4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.73Zd0hd3Yq secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.leAFVmubK4 + rm /tmp/tmp.73Zd0hd3Yq /tmp/tmp.leAFVmubK4 + return 0 + deploy_version_service + desc 'install version service' + set +o xtrace ----------------------------------------------------------------------------------- install version service ----------------------------------------------------------------------------------- + kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json ++ mktemp + local LAST_OUT=/tmp/tmp.nIW5Dc07iG ++ mktemp + local LAST_ERR=/tmp/tmp.nkHmheZGAf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nIW5Dc07iG configmap/versions created + cat /tmp/tmp.nkHmheZGAf + rm /tmp/tmp.nIW5Dc07iG /tmp/tmp.nkHmheZGAf + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Yvba6xh5kB ++ mktemp + local LAST_ERR=/tmp/tmp.cRD4mlSQ8T + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Yvba6xh5kB deployment.apps/version-service created service/version-service created + cat /tmp/tmp.cRD4mlSQ8T + rm /tmp/tmp.Yvba6xh5kB /tmp/tmp.cRD4mlSQ8T + return 0 + sleep 10 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.Vgs1uQu5Xh ++ mktemp + local LAST_ERR=/tmp/tmp.dgv7cxBvSO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Vgs1uQu5Xh namespace/cert-manager created + cat /tmp/tmp.dgv7cxBvSO + rm /tmp/tmp.Vgs1uQu5Xh /tmp/tmp.dgv7cxBvSO + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.maXmXJCfaW ++ mktemp + local LAST_ERR=/tmp/tmp.r36ZylcSUd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.maXmXJCfaW namespace/cert-manager labeled + cat /tmp/tmp.r36ZylcSUd + rm /tmp/tmp.maXmXJCfaW /tmp/tmp.r36ZylcSUd + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.wZOJ2E8oQT ++ mktemp + local LAST_ERR=/tmp/tmp.1e7MNd4kDq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wZOJ2E8oQT namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.1e7MNd4kDq Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.wZOJ2E8oQT /tmp/tmp.1e7MNd4kDq + return 0 + '[' '' == 4.10 ']' + kubectl_bin -n cert-manager rollout status deployment/cert-manager --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.Oglt1l5Rly ++ mktemp + local LAST_ERR=/tmp/tmp.8Fe0WvsgBf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n cert-manager rollout status deployment/cert-manager --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Oglt1l5Rly deployment "cert-manager" successfully rolled out + cat /tmp/tmp.8Fe0WvsgBf + rm /tmp/tmp.Oglt1l5Rly /tmp/tmp.8Fe0WvsgBf + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.ZkJviIjuLH ++ mktemp + local LAST_ERR=/tmp/tmp.p2ClPg3Hyc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZkJviIjuLH deployment "cert-manager-cainjector" successfully rolled out + cat /tmp/tmp.p2ClPg3Hyc + rm /tmp/tmp.ZkJviIjuLH /tmp/tmp.p2ClPg3Hyc + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.IUm7pcpGAp ++ mktemp + local LAST_ERR=/tmp/tmp.f8bKkwXsAW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IUm7pcpGAp Waiting for deployment "cert-manager-webhook" rollout to finish: 0 of 1 updated replicas are available... deployment "cert-manager-webhook" successfully rolled out + cat /tmp/tmp.f8bKkwXsAW + rm /tmp/tmp.IUm7pcpGAp /tmp/tmp.f8bKkwXsAW + return 0 + log 'waiting for cert-manager webhook caBundle to be injected' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:19:16+0000]' waiting for cert-manager webhook caBundle to be injected [2026-07-02T04:19:16+0000] waiting for cert-manager webhook caBundle to be injected ++ date +%s + local deadline=1782966076 + kubectl_bin get validatingwebhookconfigurations cert-manager-webhook -o 'jsonpath={.webhooks[0].clientConfig.caBundle}' + grep -q . + log 'probing cert-manager webhook' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:19:17+0000]' probing cert-manager webhook [2026-07-02T04:19:17+0000] probing cert-manager webhook + local probe_file ++ mktemp + probe_file=/tmp/tmp.CVQrLzdFdH + cat + local retries=0 + kubectl_bin apply -f /tmp/tmp.CVQrLzdFdH --dry-run=server + rm -f /tmp/tmp.CVQrLzdFdH ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ tail -n3 ++ head -n1 ++ sort -V ++ grep :8.0 +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++++ grep -c percona-xtradb-cluster-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ head -1 +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VRoTwLDVyC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wjvXwMPxZJ +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.VRoTwLDVyC +++ cat /tmp/tmp.wjvXwMPxZJ +++ rm /tmp/tmp.VRoTwLDVyC /tmp/tmp.wjvXwMPxZJ +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-59848dbdcc-crj4g -n pxc-operator -- curl -s http://version-service.smart-update1-19768.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.jqMJwsVVlF +++ mktemp ++ local LAST_ERR=/tmp/tmp.VJ6qoMFyvU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-59848dbdcc-crj4g -n pxc-operator -- curl -s http://version-service.smart-update1-19768.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jqMJwsVVlF ++ cat /tmp/tmp.VJ6qoMFyvU Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.jqMJwsVVlF /tmp/tmp.VJ6qoMFyvU ++ return 0 + IMAGE_PXC=percona/percona-xtradb-cluster:8.0.41-32.1 + desc 'patch crd' + set +o xtrace ----------------------------------------------------------------------------------- patch crd ----------------------------------------------------------------------------------- + kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' ++ mktemp + local LAST_OUT=/tmp/tmp.vceH1YwZ4R ++ mktemp + local LAST_ERR=/tmp/tmp.wlZc1jRgjB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vceH1YwZ4R customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched + cat /tmp/tmp.wlZc1jRgjB + rm /tmp/tmp.vceH1YwZ4R /tmp/tmp.wlZc1jRgjB + return 0 + desc 'Updating ProxySQL PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- Updating ProxySQL PXC cluster ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/conf/smart-update.yml /tmp/tmp.Cj53gF286S/smart-update.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858"' /tmp/tmp.Cj53gF286S/smart-update.yml + spinup_pxc smart-update /tmp/tmp.Cj53gF286S/smart-update.yml + local cluster=smart-update + local config=/tmp/tmp.Cj53gF286S/smart-update.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.8xXTVqjTaU ++ mktemp + local LAST_ERR=/tmp/tmp.xZk7h5Vvjo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8xXTVqjTaU secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.xZk7h5Vvjo + rm /tmp/tmp.8xXTVqjTaU /tmp/tmp.xZk7h5Vvjo + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_OUT=/tmp/tmp.IOrfW6xxrr + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-19768~ + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.unpmnEEZ0W + local exit_status=0 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IOrfW6xxrr deployment.apps/pxc-client created + cat /tmp/tmp.unpmnEEZ0W + rm /tmp/tmp.IOrfW6xxrr /tmp/tmp.unpmnEEZ0W + return 0 + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + apply_config /tmp/tmp.Cj53gF286S/smart-update.yml + local config_file=/tmp/tmp.Cj53gF286S/smart-update.yml + local pvc_name= + '[' -z '' ']' + cat_config /tmp/tmp.Cj53gF286S/smart-update.yml '' + kubectl_bin apply -f - + local input_file=/tmp/tmp.Cj53gF286S/smart-update.yml + local pvc_name= + cat /tmp/tmp.Cj53gF286S/smart-update.yml ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + local LAST_OUT=/tmp/tmp.z6bQWAe5J0 + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-19768~ + local LAST_ERR=/tmp/tmp.ad90PyMEgV + local exit_status=0 + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.z6bQWAe5J0 perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.ad90PyMEgV + rm /tmp/tmp.z6bQWAe5J0 /tmp/tmp.ad90PyMEgV + return 0 + desc 'waiting pods to be running for smart-update' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for smart-update ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3g9K6nxjq4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gdyXCFZ5R7 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.3g9K6nxjq4 +++ cat /tmp/tmp.gdyXCFZ5R7 +++ rm /tmp/tmp.3g9K6nxjq4 /tmp/tmp.gdyXCFZ5R7 +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TDrQDFL96M ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9TR5riXeJk +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.TDrQDFL96M +++ cat /tmp/tmp.9TR5riXeJk +++ rm /tmp/tmp.TDrQDFL96M /tmp/tmp.9TR5riXeJk +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-proxysql ++ return + local proxy=smart-update-proxysql ++ get_proxy_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cT9wsZoVED ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cofcXe6jMB +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.cT9wsZoVED +++ cat /tmp/tmp.cofcXe6jMB +++ rm /tmp/tmp.cT9wsZoVED /tmp/tmp.cofcXe6jMB +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cyzJlidmjb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mJ1hr6M9kB +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.cyzJlidmjb +++ cat /tmp/tmp.mJ1hr6M9kB +++ rm /tmp/tmp.cyzJlidmjb /tmp/tmp.mJ1hr6M9kB +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2uhiYdDLpI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SHcamlIszu +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.2uhiYdDLpI +++ cat /tmp/tmp.SHcamlIszu +++ rm /tmp/tmp.2uhiYdDLpI /tmp/tmp.SHcamlIszu +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z smart-update-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for smart-update-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for smart-update-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace Error from server (NotFound): pods "smart-update-pxc-0" not found waiting for pod/smart-update-pxc-0 to become Ready.......................Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-1 + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + wait_for_running smart-update-proxysql 2 + local name=smart-update-proxysql + local size=2 + local last_pod + local max_retry=480 + [[ -z smart-update-proxysql ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for smart-update-proxysql' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for smart-update-proxysql ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-proxysql-0 480 + local pod=smart-update-proxysql-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-proxysql-0 ++ grep -E '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/smart-update-proxysql-0 condition met E0702 04:22:43.447952 23592 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-proxysql-0&resourceVersion=1782966162625473000&timeoutSeconds=366&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-proxysql-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-proxysql-1 480 + local pod=smart-update-proxysql-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-proxysql-1 + local container=proxysql + set +o xtrace pod/smart-update-proxysql-1 condition met E0702 04:22:52.836538 24980 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-proxysql-1&resourceVersion=1782966172625545000&timeoutSeconds=377&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-proxysql-1 to become Ready.Ok + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vYChAh5KWv +++ mktemp ++ local LAST_ERR=/tmp/tmp.Yj3plzBIx7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vYChAh5KWv ++ cat /tmp/tmp.Yj3plzBIx7 ++ rm /tmp/tmp.vYChAh5KWv /tmp/tmp.Yj3plzBIx7 ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w4hGubyk5I +++ mktemp ++ local LAST_ERR=/tmp/tmp.mbOzoquq44 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.w4hGubyk5I ++ cat /tmp/tmp.mbOzoquq44 ++ rm /tmp/tmp.w4hGubyk5I /tmp/tmp.mbOzoquq44 ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.YBVyhytusm ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.YOq4rDPENr +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.YBVyhytusm +++++ cat /tmp/tmp.YOq4rDPENr +++++ rm /tmp/tmp.YBVyhytusm /tmp/tmp.YOq4rDPENr +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.EMGrGsVCa4 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.lz0fJME705 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.EMGrGsVCa4 +++++ cat /tmp/tmp.lz0fJME705 +++++ rm /tmp/tmp.EMGrGsVCa4 /tmp/tmp.lz0fJME705 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-proxysql ++++ return +++ local cluster_proxy=smart-update-proxysql +++ echo proxysql ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3mLlWsYP9T +++ mktemp ++ local LAST_ERR=/tmp/tmp.n37lT6Waxa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3mLlWsYP9T ++ cat /tmp/tmp.n37lT6Waxa ++ rm /tmp/tmp.3mLlWsYP9T /tmp/tmp.n37lT6Waxa ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XePythS9mb +++ mktemp ++ local LAST_ERR=/tmp/tmp.771G3GnSwU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XePythS9mb ++ cat /tmp/tmp.771G3GnSwU ++ rm /tmp/tmp.XePythS9mb /tmp/tmp.771G3GnSwU ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z7slipBUvL +++ mktemp ++ local LAST_ERR=/tmp/tmp.LIIs5Hw1B2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Z7slipBUvL ++ cat /tmp/tmp.LIIs5Hw1B2 ++ rm /tmp/tmp.Z7slipBUvL /tmp/tmp.LIIs5Hw1B2 ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6666d5df5c-h5cpp ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:23:28.299681 30412 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782966207178933000&timeoutSeconds=339&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data smart-update 100500 no-proxy -P3306 + local cluster=smart-update + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.L5DAKXTNLO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RiUTa7A8de +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.L5DAKXTNLO +++ cat /tmp/tmp.RiUTa7A8de +++ rm /tmp/tmp.L5DAKXTNLO /tmp/tmp.RiUTa7A8de +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v02UkhoUO8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ib0ErU3c7V ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.v02UkhoUO8 ++ cat /tmp/tmp.Ib0ErU3c7V ++ rm /tmp/tmp.v02UkhoUO8 /tmp/tmp.Ib0ErU3c7V ++ return 0 + root_pass=root_password ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RNps7ysqJI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0VbHbHsw3L +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.RNps7ysqJI +++ cat /tmp/tmp.0VbHbHsw3L +++ rm /tmp/tmp.RNps7ysqJI /tmp/tmp.0VbHbHsw3L +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kKgL70Xvql ++++ mktemp +++ local LAST_ERR=/tmp/tmp.QrzqRhevL5 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.kKgL70Xvql +++ cat /tmp/tmp.QrzqRhevL5 +++ rm /tmp/tmp.kKgL70Xvql /tmp/tmp.QrzqRhevL5 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-proxysql ++ return + proxy=smart-update-proxysql + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h smart-update-pxc-0.smart-update-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pc23iEL68K +++ mktemp ++ local LAST_ERR=/tmp/tmp.6JmEkFr5EQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Pc23iEL68K ++ cat /tmp/tmp.6JmEkFr5EQ ++ rm /tmp/tmp.Pc23iEL68K /tmp/tmp.6JmEkFr5EQ ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-h5cpp + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:23:48.315068 817 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782966227541330000&timeoutSeconds=479&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data smart-update select-1 no-proxy -P3306 + local cluster=smart-update + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.nI5bZjrDtH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oSFrKwTqsT +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.nI5bZjrDtH +++ cat /tmp/tmp.oSFrKwTqsT +++ rm /tmp/tmp.nI5bZjrDtH /tmp/tmp.oSFrKwTqsT +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tOCpuj5Bbe +++ mktemp ++ local LAST_ERR=/tmp/tmp.55aiKTtb6K ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tOCpuj5Bbe ++ cat /tmp/tmp.55aiKTtb6K ++ rm /tmp/tmp.tOCpuj5Bbe /tmp/tmp.55aiKTtb6K ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.FOzEcPBAKz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sbhpDaWn9d +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.FOzEcPBAKz +++ cat /tmp/tmp.sbhpDaWn9d +++ rm /tmp/tmp.FOzEcPBAKz /tmp/tmp.sbhpDaWn9d +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.p7oI9lDE4M +++ mktemp ++ local LAST_ERR=/tmp/tmp.OWDFlxFCyt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.p7oI9lDE4M ++ cat /tmp/tmp.OWDFlxFCyt ++ rm /tmp/tmp.p7oI9lDE4M /tmp/tmp.OWDFlxFCyt ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-h5cpp ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:24:32.900652 7103 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782966270877798000&timeoutSeconds=587&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ikGx4ab0Bi +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZGobmqgh8T ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ikGx4ab0Bi ++ cat /tmp/tmp.ZGobmqgh8T ++ rm /tmp/tmp.ikGx4ab0Bi /tmp/tmp.ZGobmqgh8T ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6666d5df5c-h5cpp + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:24:39.456814 8195 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782966278636332000&timeoutSeconds=489&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OcVLR7qcmq +++ mktemp ++ local LAST_ERR=/tmp/tmp.SOMhsrWGpA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OcVLR7qcmq ++ cat /tmp/tmp.SOMhsrWGpA ++ rm /tmp/tmp.OcVLR7qcmq /tmp/tmp.SOMhsrWGpA ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-h5cpp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:24:49.084650 9477 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782966287282034000&timeoutSeconds=514&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + grep -E -o 'early-plugin-load=keyring_\w+.so' + kubectl exec smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + return 1 +++ get_proxy smart-update +++ local target_cluster=smart-update ++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.hytUoeuPfL +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.CEze54GxAt ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.hytUoeuPfL ++++ cat /tmp/tmp.CEze54GxAt ++++ rm /tmp/tmp.hytUoeuPfL /tmp/tmp.CEze54GxAt ++++ return 0 +++ [[ '' == \t\r\u\e ]] ++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.v1ukXK3kD5 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.jxzqtjaIUA ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.v1ukXK3kD5 ++++ cat /tmp/tmp.jxzqtjaIUA ++++ rm /tmp/tmp.v1ukXK3kD5 /tmp/tmp.jxzqtjaIUA ++++ return 0 +++ [[ true == \t\r\u\e ]] +++ echo smart-update-proxysql +++ return ++ get_proxy_primary '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=smart-update-proxysql-0 +++ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 proxysql +++ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' +++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' +++ local pod=smart-update-proxysql-0 +++ local container_name=proxysql +++ set +o xtrace ++ local ip=smart-update-pxc-2.smart-update-pxc.smart-update1-19768.svc.cluster.local +++ echo smart-update-pxc-2.smart-update-pxc.smart-update1-19768.svc.cluster.local +++ wc -l ++ '[' 1 '!=' 1 ']' ++ cut -d. -f1 ++ echo smart-update-pxc-2.smart-update-pxc.smart-update1-19768.svc.cluster.local + initial_primary=smart-update-pxc-2 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.Fo2cbW2Ahz ++ mktemp + local LAST_ERR=/tmp/tmp.d4atD1gE0e + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Fo2cbW2Ahz perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.d4atD1gE0e + rm /tmp/tmp.Fo2cbW2Ahz /tmp/tmp.d4atD1gE0e + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-2 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-2 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update.............................. smart-update-pxc-2 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Eer8RTwKaU +++ mktemp ++ local LAST_ERR=/tmp/tmp.mzuvP9U8js ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Eer8RTwKaU ++ cat /tmp/tmp.mzuvP9U8js ++ rm /tmp/tmp.Eer8RTwKaU /tmp/tmp.mzuvP9U8js ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ylqTbR4BdY +++ mktemp ++ local LAST_ERR=/tmp/tmp.IBvkvxHL5T ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ylqTbR4BdY ++ cat /tmp/tmp.IBvkvxHL5T ++ rm /tmp/tmp.ylqTbR4BdY /tmp/tmp.IBvkvxHL5T ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.adiAuMRRQq ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.c4QTB8Yhrh +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.adiAuMRRQq +++++ cat /tmp/tmp.c4QTB8Yhrh +++++ rm /tmp/tmp.adiAuMRRQq /tmp/tmp.c4QTB8Yhrh +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.OVxFNLOI7I ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.oV4Y6D1ROi +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.OVxFNLOI7I +++++ cat /tmp/tmp.oV4Y6D1ROi +++++ rm /tmp/tmp.OVxFNLOI7I /tmp/tmp.oV4Y6D1ROi +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-proxysql ++++ return +++ local cluster_proxy=smart-update-proxysql +++ echo proxysql ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LtBQe6D1Zh +++ mktemp ++ local LAST_ERR=/tmp/tmp.6UWpYdppSN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LtBQe6D1Zh ++ cat /tmp/tmp.6UWpYdppSN ++ rm /tmp/tmp.LtBQe6D1Zh /tmp/tmp.6UWpYdppSN ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3NejvFILuX +++ mktemp ++ local LAST_ERR=/tmp/tmp.MfPvzYv89Y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3NejvFILuX ++ cat /tmp/tmp.MfPvzYv89Y ++ rm /tmp/tmp.3NejvFILuX /tmp/tmp.MfPvzYv89Y ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AjrXuaNxG4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.3yqWYgyjLc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AjrXuaNxG4 ++ cat /tmp/tmp.3yqWYgyjLc ++ rm /tmp/tmp.AjrXuaNxG4 /tmp/tmp.3yqWYgyjLc ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.lIl3VasfpV ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.uW3u6w2NNF +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.lIl3VasfpV +++++ cat /tmp/tmp.uW3u6w2NNF +++++ rm /tmp/tmp.lIl3VasfpV /tmp/tmp.uW3u6w2NNF +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.3XJ1u5MbuA ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.B5VM92JYUT +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.3XJ1u5MbuA +++++ cat /tmp/tmp.B5VM92JYUT +++++ rm /tmp/tmp.3XJ1u5MbuA /tmp/tmp.B5VM92JYUT +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-proxysql ++++ return +++ local cluster_proxy=smart-update-proxysql +++ echo proxysql ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hWycQj2hWJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.J2hbrqNfqc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hWycQj2hWJ ++ cat /tmp/tmp.J2hbrqNfqc ++ rm /tmp/tmp.hWycQj2hWJ /tmp/tmp.J2hbrqNfqc ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NQWuyMo5g8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.7F11UJAhBU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NQWuyMo5g8 ++ cat /tmp/tmp.7F11UJAhBU ++ rm /tmp/tmp.NQWuyMo5g8 /tmp/tmp.7F11UJAhBU ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U29pzzdNwS +++ mktemp ++ local LAST_ERR=/tmp/tmp.PaNMqslhEH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.U29pzzdNwS ++ cat /tmp/tmp.PaNMqslhEH ++ rm /tmp/tmp.U29pzzdNwS /tmp/tmp.PaNMqslhEH ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.faBMCNj2rO ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.l6teMVyVtg +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.faBMCNj2rO +++++ cat /tmp/tmp.l6teMVyVtg +++++ rm /tmp/tmp.faBMCNj2rO /tmp/tmp.l6teMVyVtg +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.opUkZKHQVr ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.tWNRik4tCw +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.opUkZKHQVr +++++ cat /tmp/tmp.tWNRik4tCw +++++ rm /tmp/tmp.opUkZKHQVr /tmp/tmp.tWNRik4tCw +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-proxysql ++++ return +++ local cluster_proxy=smart-update-proxysql +++ echo proxysql ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oY2iISLMHj +++ mktemp ++ local LAST_ERR=/tmp/tmp.0Z6dQaxRZC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oY2iISLMHj ++ cat /tmp/tmp.0Z6dQaxRZC ++ rm /tmp/tmp.oY2iISLMHj /tmp/tmp.0Z6dQaxRZC ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WuOXXR7VZN +++ mktemp ++ local LAST_ERR=/tmp/tmp.Q6iqLN8wgT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WuOXXR7VZN ++ cat /tmp/tmp.Q6iqLN8wgT ++ rm /tmp/tmp.WuOXXR7VZN /tmp/tmp.Q6iqLN8wgT ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dcQLGaQT3M +++ mktemp ++ local LAST_ERR=/tmp/tmp.jT9dk8qe7z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dcQLGaQT3M ++ cat /tmp/tmp.jT9dk8qe7z ++ rm /tmp/tmp.dcQLGaQT3M /tmp/tmp.jT9dk8qe7z ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.1oIytR6opG ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Ifal047cg2 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.1oIytR6opG +++++ cat /tmp/tmp.Ifal047cg2 +++++ rm /tmp/tmp.1oIytR6opG /tmp/tmp.Ifal047cg2 +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Xi5OYULKkd ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.FzrFybYyur +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Xi5OYULKkd +++++ cat /tmp/tmp.FzrFybYyur +++++ rm /tmp/tmp.Xi5OYULKkd /tmp/tmp.FzrFybYyur +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-proxysql ++++ return +++ local cluster_proxy=smart-update-proxysql +++ echo proxysql ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dX5RFrX6FB +++ mktemp ++ local LAST_ERR=/tmp/tmp.WuwYrgWd7y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dX5RFrX6FB ++ cat /tmp/tmp.WuwYrgWd7y ++ rm /tmp/tmp.dX5RFrX6FB /tmp/tmp.WuwYrgWd7y ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uTMs7npFjy +++ mktemp ++ local LAST_ERR=/tmp/tmp.Vi5utWq7Uy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uTMs7npFjy ++ cat /tmp/tmp.Vi5utWq7Uy ++ rm /tmp/tmp.uTMs7npFjy /tmp/tmp.Vi5utWq7Uy ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nsVPyvrOuU +++ mktemp ++ local LAST_ERR=/tmp/tmp.djL5GtjuLz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nsVPyvrOuU ++ cat /tmp/tmp.djL5GtjuLz ++ rm /tmp/tmp.nsVPyvrOuU /tmp/tmp.djL5GtjuLz ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.7gv5HZpzpj ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.UpkRLMCa2B +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.7gv5HZpzpj +++++ cat /tmp/tmp.UpkRLMCa2B +++++ rm /tmp/tmp.7gv5HZpzpj /tmp/tmp.UpkRLMCa2B +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.UiMlzhvW8Y ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.UlJBltjnBI +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.UiMlzhvW8Y +++++ cat /tmp/tmp.UlJBltjnBI +++++ rm /tmp/tmp.UiMlzhvW8Y /tmp/tmp.UlJBltjnBI +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-proxysql ++++ return +++ local cluster_proxy=smart-update-proxysql +++ echo proxysql ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2WyKJROocl +++ mktemp ++ local LAST_ERR=/tmp/tmp.ilJW1yiN1o ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2WyKJROocl ++ cat /tmp/tmp.ilJW1yiN1o ++ rm /tmp/tmp.2WyKJROocl /tmp/tmp.ilJW1yiN1o ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uQAXulU2k3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Yz6WpkbtRf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uQAXulU2k3 ++ cat /tmp/tmp.Yz6WpkbtRf ++ rm /tmp/tmp.uQAXulU2k3 /tmp/tmp.Yz6WpkbtRf ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XQLIRje0Kn +++ mktemp ++ local LAST_ERR=/tmp/tmp.VqfnupbPus ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XQLIRje0Kn ++ cat /tmp/tmp.VqfnupbPus ++ rm /tmp/tmp.XQLIRje0Kn /tmp/tmp.VqfnupbPus ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.pt6nrOxt5o ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.VlAqP671DQ +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.pt6nrOxt5o +++++ cat /tmp/tmp.VlAqP671DQ +++++ rm /tmp/tmp.pt6nrOxt5o /tmp/tmp.VlAqP671DQ +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.CiYLmoi6qc ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.mr1bu5kOhX +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.CiYLmoi6qc +++++ cat /tmp/tmp.mr1bu5kOhX +++++ rm /tmp/tmp.CiYLmoi6qc /tmp/tmp.mr1bu5kOhX +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-proxysql ++++ return +++ local cluster_proxy=smart-update-proxysql +++ echo proxysql ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oqapC8Fa8Y +++ mktemp ++ local LAST_ERR=/tmp/tmp.0y3M9U37xG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oqapC8Fa8Y ++ cat /tmp/tmp.0y3M9U37xG ++ rm /tmp/tmp.oqapC8Fa8Y /tmp/tmp.0y3M9U37xG ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fA1km70UMO +++ mktemp ++ local LAST_ERR=/tmp/tmp.eXAC5s4nN7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fA1km70UMO ++ cat /tmp/tmp.eXAC5s4nN7 ++ rm /tmp/tmp.fA1km70UMO /tmp/tmp.eXAC5s4nN7 ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6666d5df5c-h5cpp + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:29:11.527962 15491 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782966549104977000&timeoutSeconds=578&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EISsZBXBkt +++ mktemp ++ local LAST_ERR=/tmp/tmp.pz2ke115Ne ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EISsZBXBkt ++ cat /tmp/tmp.pz2ke115Ne ++ rm /tmp/tmp.EISsZBXBkt /tmp/tmp.pz2ke115Ne ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-h5cpp + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:29:24.187425 17588 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782966561660660000&timeoutSeconds=362&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.roNrWevFgZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.5L68VmiCF6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.roNrWevFgZ ++ cat /tmp/tmp.5L68VmiCF6 ++ rm /tmp/tmp.roNrWevFgZ /tmp/tmp.5L68VmiCF6 ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-h5cpp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:29:39.149154 19802 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782966576101162000&timeoutSeconds=525&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/conf/smart-update.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xKPCq59pjM ++ mktemp + local LAST_ERR=/tmp/tmp.Jp2MONTXKc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/conf/smart-update.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xKPCq59pjM perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update1-19768 namespace + cat /tmp/tmp.Jp2MONTXKc + rm /tmp/tmp.xKPCq59pjM /tmp/tmp.Jp2MONTXKc + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.FIOCQHbZEJ ++ mktemp + local LAST_ERR=/tmp/tmp.T7kbaw2gJk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FIOCQHbZEJ persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update1-19768 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update1-19768 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update1-19768 namespace persistentvolumeclaim "proxydata-smart-update-proxysql-0" deleted from smart-update1-19768 namespace persistentvolumeclaim "proxydata-smart-update-proxysql-1" deleted from smart-update1-19768 namespace + cat /tmp/tmp.T7kbaw2gJk + rm /tmp/tmp.FIOCQHbZEJ /tmp/tmp.T7kbaw2gJk + return 0 + desc 'Updating HAProxy PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- Updating HAProxy PXC cluster ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/conf/smart-update-haproxy.yml /tmp/tmp.Cj53gF286S/smart-update-haproxy.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858"' /tmp/tmp.Cj53gF286S/smart-update-haproxy.yml + spinup_pxc smart-update /tmp/tmp.Cj53gF286S/smart-update-haproxy.yml + local cluster=smart-update + local config=/tmp/tmp.Cj53gF286S/smart-update-haproxy.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.ndD2g7XJmU ++ mktemp + local LAST_ERR=/tmp/tmp.t5vTr18daw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ndD2g7XJmU secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.t5vTr18daw + rm /tmp/tmp.ndD2g7XJmU /tmp/tmp.t5vTr18daw + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-19768~ ++ mktemp + local LAST_OUT=/tmp/tmp.1Mfslwi3HI ++ mktemp + local LAST_ERR=/tmp/tmp.3WL0nOhLMz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1Mfslwi3HI deployment.apps/pxc-client unchanged + cat /tmp/tmp.3WL0nOhLMz + rm /tmp/tmp.1Mfslwi3HI /tmp/tmp.3WL0nOhLMz + return 0 + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + apply_config /tmp/tmp.Cj53gF286S/smart-update-haproxy.yml + local config_file=/tmp/tmp.Cj53gF286S/smart-update-haproxy.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.Cj53gF286S/smart-update-haproxy.yml '' + local input_file=/tmp/tmp.Cj53gF286S/smart-update-haproxy.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-19768~ + cat /tmp/tmp.Cj53gF286S/smart-update-haproxy.yml + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + local LAST_OUT=/tmp/tmp.bdCZD0ckSt ++ mktemp + local LAST_ERR=/tmp/tmp.2yOqGkVS9O + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bdCZD0ckSt perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.2yOqGkVS9O + rm /tmp/tmp.bdCZD0ckSt /tmp/tmp.2yOqGkVS9O + return 0 + desc 'waiting pods to be running for smart-update' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for smart-update ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mY7f0JKoX8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oRY5pq5H4X +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.mY7f0JKoX8 +++ cat /tmp/tmp.oRY5pq5H4X +++ rm /tmp/tmp.mY7f0JKoX8 /tmp/tmp.oRY5pq5H4X +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy ++ get_proxy_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zr72mT6Tps ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fvyaof4eQS +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.zr72mT6Tps +++ cat /tmp/tmp.fvyaof4eQS +++ rm /tmp/tmp.zr72mT6Tps /tmp/tmp.fvyaof4eQS +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pmy4dCWf6f ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TEMjSPot4x +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.pmy4dCWf6f +++ cat /tmp/tmp.TEMjSPot4x +++ rm /tmp/tmp.pmy4dCWf6f /tmp/tmp.TEMjSPot4x +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z smart-update-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for smart-update-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for smart-update-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + wait_for_running smart-update-haproxy 2 + local name=smart-update-haproxy + local size=2 + local last_pod + local max_retry=480 + [[ -z smart-update-haproxy ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for smart-update-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for smart-update-haproxy ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met E0702 04:34:59.247710 723 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-0&resourceVersion=1782966897125016000&timeoutSeconds=524&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-1 480 + local pod=smart-update-haproxy-1 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-1 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/smart-update-haproxy-1 condition met E0702 04:35:05.627802 1731 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-1&resourceVersion=1782966902125087000&timeoutSeconds=571&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-1 to become Ready.Ok + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.txMipwxZq0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.DL4C9Hs3P9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.txMipwxZq0 ++ cat /tmp/tmp.DL4C9Hs3P9 ++ rm /tmp/tmp.txMipwxZq0 /tmp/tmp.DL4C9Hs3P9 ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Gp0vSfNzGK +++ mktemp ++ local LAST_ERR=/tmp/tmp.9OngE9HIKb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Gp0vSfNzGK ++ cat /tmp/tmp.9OngE9HIKb ++ rm /tmp/tmp.Gp0vSfNzGK /tmp/tmp.9OngE9HIKb ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Xj8kahrLsF ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.K8RLWY38Ls +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Xj8kahrLsF +++++ cat /tmp/tmp.K8RLWY38Ls +++++ rm /tmp/tmp.Xj8kahrLsF /tmp/tmp.K8RLWY38Ls +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.svw7vpVcOT +++ mktemp ++ local LAST_ERR=/tmp/tmp.FLKJ781ohN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.svw7vpVcOT ++ cat /tmp/tmp.FLKJ781ohN ++ rm /tmp/tmp.svw7vpVcOT /tmp/tmp.FLKJ781ohN ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tGvEe8i3sb +++ mktemp ++ local LAST_ERR=/tmp/tmp.NyyekD7uk7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tGvEe8i3sb ++ cat /tmp/tmp.NyyekD7uk7 ++ rm /tmp/tmp.tGvEe8i3sb /tmp/tmp.NyyekD7uk7 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6yBxsWVcTt +++ mktemp ++ local LAST_ERR=/tmp/tmp.2UASIreztW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6yBxsWVcTt ++ cat /tmp/tmp.2UASIreztW ++ rm /tmp/tmp.6yBxsWVcTt /tmp/tmp.2UASIreztW ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-h5cpp ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:35:39.088587 6490 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782966937079283000&timeoutSeconds=544&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data smart-update 100500 no-proxy -P3306 + local cluster=smart-update + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Ktck9YdU1F ++++ mktemp +++ local LAST_ERR=/tmp/tmp.F2xmoKkvNH +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Ktck9YdU1F +++ cat /tmp/tmp.F2xmoKkvNH +++ rm /tmp/tmp.Ktck9YdU1F /tmp/tmp.F2xmoKkvNH +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ej5bFpWm0K +++ mktemp ++ local LAST_ERR=/tmp/tmp.KyzsWrAD14 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ej5bFpWm0K ++ cat /tmp/tmp.KyzsWrAD14 ++ rm /tmp/tmp.ej5bFpWm0K /tmp/tmp.KyzsWrAD14 ++ return 0 + root_pass=root_password ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bk7zpYqxOR ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YUZtntavUx +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.bk7zpYqxOR +++ cat /tmp/tmp.YUZtntavUx +++ rm /tmp/tmp.bk7zpYqxOR /tmp/tmp.YUZtntavUx +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + proxy=smart-update-haproxy + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h smart-update-pxc-0.smart-update-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PGQLssz7rw +++ mktemp ++ local LAST_ERR=/tmp/tmp.hdu9hTGWYt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PGQLssz7rw ++ cat /tmp/tmp.hdu9hTGWYt ++ rm /tmp/tmp.PGQLssz7rw /tmp/tmp.hdu9hTGWYt ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6666d5df5c-h5cpp ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:35:52.726384 8621 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782966950722656000&timeoutSeconds=444&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data smart-update select-1 no-proxy -P3306 + local cluster=smart-update + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Ce6egmiPw5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5vc1FoPIez +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Ce6egmiPw5 +++ cat /tmp/tmp.5vc1FoPIez +++ rm /tmp/tmp.Ce6egmiPw5 /tmp/tmp.5vc1FoPIez +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.AWGREvWBEa +++ mktemp ++ local LAST_ERR=/tmp/tmp.NZzQln5hvW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AWGREvWBEa ++ cat /tmp/tmp.NZzQln5hvW ++ rm /tmp/tmp.AWGREvWBEa /tmp/tmp.NZzQln5hvW ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GBhoh5bH2l ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uHwrIfHh5g +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.GBhoh5bH2l +++ cat /tmp/tmp.uHwrIfHh5g +++ rm /tmp/tmp.GBhoh5bH2l /tmp/tmp.uHwrIfHh5g +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EPrkkMWxK5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.MbOAv9uc2V ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EPrkkMWxK5 ++ cat /tmp/tmp.MbOAv9uc2V ++ rm /tmp/tmp.EPrkkMWxK5 /tmp/tmp.MbOAv9uc2V ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6666d5df5c-h5cpp + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:36:38.274101 14834 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782966995700904000&timeoutSeconds=555&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.63yAoQjsLG +++ mktemp ++ local LAST_ERR=/tmp/tmp.eEC0zASfah ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.63yAoQjsLG ++ cat /tmp/tmp.eEC0zASfah ++ rm /tmp/tmp.63yAoQjsLG /tmp/tmp.eEC0zASfah ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6666d5df5c-h5cpp ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:36:54.773721 17350 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967011467487000&timeoutSeconds=427&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GV8NgyUECk +++ mktemp ++ local LAST_ERR=/tmp/tmp.o7x5gtzPHH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GV8NgyUECk ++ cat /tmp/tmp.o7x5gtzPHH ++ rm /tmp/tmp.GV8NgyUECk /tmp/tmp.o7x5gtzPHH ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-h5cpp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:37:06.300561 19191 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967024351384000&timeoutSeconds=571&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + kubectl exec smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + grep -E -o 'early-plugin-load=keyring_\w+.so' + return 1 ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fsOli1pDGd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.exNsYX0KU5 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.fsOli1pDGd +++ cat /tmp/tmp.exNsYX0KU5 +++ rm /tmp/tmp.fsOli1pDGd /tmp/tmp.exNsYX0KU5 +++ return 0 ++ client_pod=pxc-client-6666d5df5c-h5cpp ++ wait_pod pxc-client-6666d5df5c-h5cpp ++ local pod=pxc-client-6666d5df5c-h5cpp ++ local max_retry=480 ++ local ns= +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ grep -E '^(pxc|proxysql)$' +++ echo pxc-client-6666d5df5c-h5cpp ++ local container= ++ set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:37:23.146929 21537 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967040208868000&timeoutSeconds=480&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.olPF4UYVSQ ++ mktemp + local LAST_ERR=/tmp/tmp.2LhPIGr9mG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.olPF4UYVSQ perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.2LhPIGr9mG + rm /tmp/tmp.olPF4UYVSQ /tmp/tmp.2LhPIGr9mG + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update............ smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tHbrXSgNFd +++ mktemp ++ local LAST_ERR=/tmp/tmp.pwqjLJiVPI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tHbrXSgNFd ++ cat /tmp/tmp.pwqjLJiVPI ++ rm /tmp/tmp.tHbrXSgNFd /tmp/tmp.pwqjLJiVPI ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sRQlBfscJj +++ mktemp ++ local LAST_ERR=/tmp/tmp.VkFsSOuqC3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sRQlBfscJj ++ cat /tmp/tmp.VkFsSOuqC3 ++ rm /tmp/tmp.sRQlBfscJj /tmp/tmp.VkFsSOuqC3 ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.tku3NTLNFB ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.rIVd70xrfQ +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.tku3NTLNFB +++++ cat /tmp/tmp.rIVd70xrfQ +++++ rm /tmp/tmp.tku3NTLNFB /tmp/tmp.rIVd70xrfQ +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vDeweIkQoP +++ mktemp ++ local LAST_ERR=/tmp/tmp.wYkBWdEECK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vDeweIkQoP ++ cat /tmp/tmp.wYkBWdEECK ++ rm /tmp/tmp.vDeweIkQoP /tmp/tmp.wYkBWdEECK ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q73yl2Fwo3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.54phHh8Csa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.q73yl2Fwo3 ++ cat /tmp/tmp.54phHh8Csa ++ rm /tmp/tmp.q73yl2Fwo3 /tmp/tmp.54phHh8Csa ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QiMxlO2nQi +++ mktemp ++ local LAST_ERR=/tmp/tmp.7C2x0XSjUU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QiMxlO2nQi ++ cat /tmp/tmp.7C2x0XSjUU ++ rm /tmp/tmp.QiMxlO2nQi /tmp/tmp.7C2x0XSjUU ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.URxDnZsVRB ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.jrNVRAwQQf +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.URxDnZsVRB +++++ cat /tmp/tmp.jrNVRAwQQf +++++ rm /tmp/tmp.URxDnZsVRB /tmp/tmp.jrNVRAwQQf +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UVgkluzmQF +++ mktemp ++ local LAST_ERR=/tmp/tmp.Y8Sv2A7k2k ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UVgkluzmQF ++ cat /tmp/tmp.Y8Sv2A7k2k ++ rm /tmp/tmp.UVgkluzmQF /tmp/tmp.Y8Sv2A7k2k ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.F2kISugtKf +++ mktemp ++ local LAST_ERR=/tmp/tmp.V9FYVnQR1H ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.F2kISugtKf ++ cat /tmp/tmp.V9FYVnQR1H ++ rm /tmp/tmp.F2kISugtKf /tmp/tmp.V9FYVnQR1H ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ALNlSpkW4h +++ mktemp ++ local LAST_ERR=/tmp/tmp.TLaRQe3OPH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ALNlSpkW4h ++ cat /tmp/tmp.TLaRQe3OPH ++ rm /tmp/tmp.ALNlSpkW4h /tmp/tmp.TLaRQe3OPH ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.L7KyvyW2dq ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.u7L9XpfnRe +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.L7KyvyW2dq +++++ cat /tmp/tmp.u7L9XpfnRe +++++ rm /tmp/tmp.L7KyvyW2dq /tmp/tmp.u7L9XpfnRe +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RpFQroAzek +++ mktemp ++ local LAST_ERR=/tmp/tmp.L8Gu72eeA6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RpFQroAzek ++ cat /tmp/tmp.L8Gu72eeA6 ++ rm /tmp/tmp.RpFQroAzek /tmp/tmp.L8Gu72eeA6 ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Gd92pOEZhu +++ mktemp ++ local LAST_ERR=/tmp/tmp.XggpDAXN9N ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Gd92pOEZhu ++ cat /tmp/tmp.XggpDAXN9N ++ rm /tmp/tmp.Gd92pOEZhu /tmp/tmp.XggpDAXN9N ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZG5IbixXFe +++ mktemp ++ local LAST_ERR=/tmp/tmp.cRG8Ziw03W ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZG5IbixXFe ++ cat /tmp/tmp.cRG8Ziw03W ++ rm /tmp/tmp.ZG5IbixXFe /tmp/tmp.cRG8Ziw03W ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.UCCYiYCFqk ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.oa7Dkpoebw +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.UCCYiYCFqk +++++ cat /tmp/tmp.oa7Dkpoebw +++++ rm /tmp/tmp.UCCYiYCFqk /tmp/tmp.oa7Dkpoebw +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FB0cw7iNvV +++ mktemp ++ local LAST_ERR=/tmp/tmp.Fkt27XB5J1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FB0cw7iNvV ++ cat /tmp/tmp.Fkt27XB5J1 ++ rm /tmp/tmp.FB0cw7iNvV /tmp/tmp.Fkt27XB5J1 ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oRAK1rAPTz +++ mktemp ++ local LAST_ERR=/tmp/tmp.dA7ZQPj6bb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oRAK1rAPTz ++ cat /tmp/tmp.dA7ZQPj6bb ++ rm /tmp/tmp.oRAK1rAPTz /tmp/tmp.dA7ZQPj6bb ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hdwrryYsrR +++ mktemp ++ local LAST_ERR=/tmp/tmp.qVQBJeaoM3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hdwrryYsrR ++ cat /tmp/tmp.qVQBJeaoM3 ++ rm /tmp/tmp.hdwrryYsrR /tmp/tmp.qVQBJeaoM3 ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.jiRXDnVQTU ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.JvcAFtCt1n +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.jiRXDnVQTU +++++ cat /tmp/tmp.JvcAFtCt1n +++++ rm /tmp/tmp.jiRXDnVQTU /tmp/tmp.JvcAFtCt1n +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2rJOArWGQG +++ mktemp ++ local LAST_ERR=/tmp/tmp.pSxhPhYnQK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2rJOArWGQG ++ cat /tmp/tmp.pSxhPhYnQK ++ rm /tmp/tmp.2rJOArWGQG /tmp/tmp.pSxhPhYnQK ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ASW2kAjlQW +++ mktemp ++ local LAST_ERR=/tmp/tmp.DFCty7XJjF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ASW2kAjlQW ++ cat /tmp/tmp.DFCty7XJjF ++ rm /tmp/tmp.ASW2kAjlQW /tmp/tmp.DFCty7XJjF ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4jx4M1PnNb +++ mktemp ++ local LAST_ERR=/tmp/tmp.HCdi3MAawt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4jx4M1PnNb ++ cat /tmp/tmp.HCdi3MAawt ++ rm /tmp/tmp.4jx4M1PnNb /tmp/tmp.HCdi3MAawt ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.ffVJWgENaw ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.QL2z562Gh2 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.ffVJWgENaw +++++ cat /tmp/tmp.QL2z562Gh2 +++++ rm /tmp/tmp.ffVJWgENaw /tmp/tmp.QL2z562Gh2 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MDUPFanIxn +++ mktemp ++ local LAST_ERR=/tmp/tmp.9rqQ4iGJ3Y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MDUPFanIxn ++ cat /tmp/tmp.9rqQ4iGJ3Y ++ rm /tmp/tmp.MDUPFanIxn /tmp/tmp.9rqQ4iGJ3Y ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.098YxSwXeC +++ mktemp ++ local LAST_ERR=/tmp/tmp.HyA7OxF2SN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.098YxSwXeC ++ cat /tmp/tmp.HyA7OxF2SN ++ rm /tmp/tmp.098YxSwXeC /tmp/tmp.HyA7OxF2SN ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Kq33bGEXbO +++ mktemp ++ local LAST_ERR=/tmp/tmp.wqkYqGNZlF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Kq33bGEXbO ++ cat /tmp/tmp.wqkYqGNZlF ++ rm /tmp/tmp.Kq33bGEXbO /tmp/tmp.wqkYqGNZlF ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.L7Chsq8Q6c ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Tq9hxc7wii +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.L7Chsq8Q6c +++++ cat /tmp/tmp.Tq9hxc7wii +++++ rm /tmp/tmp.L7Chsq8Q6c /tmp/tmp.Tq9hxc7wii +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.licAaKDQ2Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.doZnsK7gSr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.licAaKDQ2Q ++ cat /tmp/tmp.doZnsK7gSr ++ rm /tmp/tmp.licAaKDQ2Q /tmp/tmp.doZnsK7gSr ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d8IBZ18uEs +++ mktemp ++ local LAST_ERR=/tmp/tmp.aWf2OuVEMJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.d8IBZ18uEs ++ cat /tmp/tmp.aWf2OuVEMJ ++ rm /tmp/tmp.d8IBZ18uEs /tmp/tmp.aWf2OuVEMJ ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dMyzKkXoBJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.n2MmpUD0Vy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dMyzKkXoBJ ++ cat /tmp/tmp.n2MmpUD0Vy ++ rm /tmp/tmp.dMyzKkXoBJ /tmp/tmp.n2MmpUD0Vy ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.1Bp28NHBKu ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.UOwafTw542 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.1Bp28NHBKu +++++ cat /tmp/tmp.UOwafTw542 +++++ rm /tmp/tmp.1Bp28NHBKu /tmp/tmp.UOwafTw542 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vVdqAfuUGE +++ mktemp ++ local LAST_ERR=/tmp/tmp.ERYvvjXguB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vVdqAfuUGE ++ cat /tmp/tmp.ERYvvjXguB ++ rm /tmp/tmp.vVdqAfuUGE /tmp/tmp.ERYvvjXguB ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5UGVQNXCjJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.tuHm5ctj66 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5UGVQNXCjJ ++ cat /tmp/tmp.tuHm5ctj66 ++ rm /tmp/tmp.5UGVQNXCjJ /tmp/tmp.tuHm5ctj66 ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CA0Fa8E7ZL +++ mktemp ++ local LAST_ERR=/tmp/tmp.GAKEBNqlNt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CA0Fa8E7ZL ++ cat /tmp/tmp.GAKEBNqlNt ++ rm /tmp/tmp.CA0Fa8E7ZL /tmp/tmp.GAKEBNqlNt ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Gs9pA0YeUN ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.f29Sh7MBh0 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Gs9pA0YeUN +++++ cat /tmp/tmp.f29Sh7MBh0 +++++ rm /tmp/tmp.Gs9pA0YeUN /tmp/tmp.f29Sh7MBh0 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Iwf4of7sD8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JGbCKkq92I ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Iwf4of7sD8 ++ cat /tmp/tmp.JGbCKkq92I ++ rm /tmp/tmp.Iwf4of7sD8 /tmp/tmp.JGbCKkq92I ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SpetjpDvXe +++ mktemp ++ local LAST_ERR=/tmp/tmp.4pjQC6lwo0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SpetjpDvXe ++ cat /tmp/tmp.4pjQC6lwo0 ++ rm /tmp/tmp.SpetjpDvXe /tmp/tmp.4pjQC6lwo0 ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.S358BxEq27 +++ mktemp ++ local LAST_ERR=/tmp/tmp.uGDbd8LGAC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.S358BxEq27 ++ cat /tmp/tmp.uGDbd8LGAC ++ rm /tmp/tmp.S358BxEq27 /tmp/tmp.uGDbd8LGAC ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.O0FFdtt29a ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ZQ91xhJPAP +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.O0FFdtt29a +++++ cat /tmp/tmp.ZQ91xhJPAP +++++ rm /tmp/tmp.O0FFdtt29a /tmp/tmp.ZQ91xhJPAP +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n2sVr3b6gO +++ mktemp ++ local LAST_ERR=/tmp/tmp.Vl27tXCBRS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.n2sVr3b6gO ++ cat /tmp/tmp.Vl27tXCBRS ++ rm /tmp/tmp.n2sVr3b6gO /tmp/tmp.Vl27tXCBRS ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7f6H9trqVk +++ mktemp ++ local LAST_ERR=/tmp/tmp.wFZbH9EvGh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7f6H9trqVk ++ cat /tmp/tmp.wFZbH9EvGh ++ rm /tmp/tmp.7f6H9trqVk /tmp/tmp.wFZbH9EvGh ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-h5cpp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:40:51.577864 19925 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967249953711000&timeoutSeconds=311&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.u3M61TliHd +++ mktemp ++ local LAST_ERR=/tmp/tmp.SXPWCYXUTL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u3M61TliHd ++ cat /tmp/tmp.SXPWCYXUTL ++ rm /tmp/tmp.u3M61TliHd /tmp/tmp.SXPWCYXUTL ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-h5cpp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:40:58.817093 20933 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967256941861000&timeoutSeconds=456&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BinfchEin6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RUocgxE5hW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BinfchEin6 ++ cat /tmp/tmp.RUocgxE5hW ++ rm /tmp/tmp.BinfchEin6 /tmp/tmp.RUocgxE5hW ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-h5cpp ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:41:06.465976 22148 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967265699457000&timeoutSeconds=312&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + kubectl_bin delete -f /tmp/tmp.Cj53gF286S/smart-update-haproxy.yml ++ mktemp + local LAST_OUT=/tmp/tmp.8i55mHY2JX ++ mktemp + local LAST_ERR=/tmp/tmp.X0nd9rpbD1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.Cj53gF286S/smart-update-haproxy.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8i55mHY2JX perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update1-19768 namespace + cat /tmp/tmp.X0nd9rpbD1 + rm /tmp/tmp.8i55mHY2JX /tmp/tmp.X0nd9rpbD1 + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.Sz2tna5yZm ++ mktemp + local LAST_ERR=/tmp/tmp.B1aGS19ZxB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Sz2tna5yZm persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update1-19768 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update1-19768 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update1-19768 namespace + cat /tmp/tmp.B1aGS19ZxB + rm /tmp/tmp.Sz2tna5yZm /tmp/tmp.B1aGS19ZxB + return 0 + desc 'Updating PXC cluster with version service available but disabled' + set +o xtrace ----------------------------------------------------------------------------------- Updating PXC cluster with version service available but disabled ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/conf/smart-update-version-service-reachable.yml /tmp/tmp.Cj53gF286S/smart-update-version-service-reachable.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858"' /tmp/tmp.Cj53gF286S/smart-update-version-service-reachable.yml + spinup_pxc smart-update /tmp/tmp.Cj53gF286S/smart-update-version-service-reachable.yml + local cluster=smart-update + local config=/tmp/tmp.Cj53gF286S/smart-update-version-service-reachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.kMzTbq5iMR ++ mktemp + local LAST_ERR=/tmp/tmp.onjczJxrHJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kMzTbq5iMR secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.onjczJxrHJ + rm /tmp/tmp.kMzTbq5iMR /tmp/tmp.onjczJxrHJ + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-19768~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_OUT=/tmp/tmp.YBXQPztwrG + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + local LAST_ERR=/tmp/tmp.YtaUXiTmEJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YBXQPztwrG deployment.apps/pxc-client unchanged + cat /tmp/tmp.YtaUXiTmEJ + rm /tmp/tmp.YBXQPztwrG /tmp/tmp.YtaUXiTmEJ + return 0 + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + apply_config /tmp/tmp.Cj53gF286S/smart-update-version-service-reachable.yml + local config_file=/tmp/tmp.Cj53gF286S/smart-update-version-service-reachable.yml + local pvc_name= + '[' -z '' ']' + cat_config /tmp/tmp.Cj53gF286S/smart-update-version-service-reachable.yml '' + kubectl_bin apply -f - + local input_file=/tmp/tmp.Cj53gF286S/smart-update-version-service-reachable.yml + local pvc_name= + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + cat /tmp/tmp.Cj53gF286S/smart-update-version-service-reachable.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2538-70c04858#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.dq4GHPhhx2 + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-19768~ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + local LAST_ERR=/tmp/tmp.oxkJ5sVj8M + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dq4GHPhhx2 perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.oxkJ5sVj8M + rm /tmp/tmp.dq4GHPhhx2 /tmp/tmp.oxkJ5sVj8M + return 0 + desc 'waiting pods to be running for smart-update' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for smart-update ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7zog8PSkkN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.37yW17arXk +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.7zog8PSkkN +++ cat /tmp/tmp.37yW17arXk +++ rm /tmp/tmp.7zog8PSkkN /tmp/tmp.37yW17arXk +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy ++ get_proxy_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7DxRNZQ7qC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vig12bKV9d +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.7DxRNZQ7qC +++ cat /tmp/tmp.vig12bKV9d +++ rm /tmp/tmp.7DxRNZQ7qC /tmp/tmp.vig12bKV9d +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0L0J1rWXuQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YbhiSm9s2E +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.0L0J1rWXuQ +++ cat /tmp/tmp.YbhiSm9s2E +++ rm /tmp/tmp.0L0J1rWXuQ /tmp/tmp.YbhiSm9s2E +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z smart-update-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for smart-update-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for smart-update-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-0 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + wait_for_running smart-update-haproxy 2 + local name=smart-update-haproxy + local size=2 + local last_pod + local max_retry=480 + [[ -z smart-update-haproxy ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for smart-update-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for smart-update-haproxy ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-haproxy-0 + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met E0702 04:46:22.036619 4335 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-0&resourceVersion=1782967578210170000&timeoutSeconds=455&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-1 480 + local pod=smart-update-haproxy-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-haproxy-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/smart-update-haproxy-1 condition met E0702 04:46:35.026751 6317 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsmart-update-haproxy-1&resourceVersion=1782967593210289000&timeoutSeconds=477&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/smart-update-haproxy-1 to become Ready.Ok + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d2QHJs9Z8T +++ mktemp ++ local LAST_ERR=/tmp/tmp.y7qhgCKeJA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.d2QHJs9Z8T ++ cat /tmp/tmp.y7qhgCKeJA ++ rm /tmp/tmp.d2QHJs9Z8T /tmp/tmp.y7qhgCKeJA ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.p4Eh4Kkz9R +++ mktemp ++ local LAST_ERR=/tmp/tmp.6FUZ6h53Hz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.p4Eh4Kkz9R ++ cat /tmp/tmp.6FUZ6h53Hz ++ rm /tmp/tmp.p4Eh4Kkz9R /tmp/tmp.6FUZ6h53Hz ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.67LyYvmbvl ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.77qqaZCy6A +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.67LyYvmbvl +++++ cat /tmp/tmp.77qqaZCy6A +++++ rm /tmp/tmp.67LyYvmbvl /tmp/tmp.77qqaZCy6A +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2gmok3d5BO +++ mktemp ++ local LAST_ERR=/tmp/tmp.yhabkAjCqE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2gmok3d5BO ++ cat /tmp/tmp.yhabkAjCqE ++ rm /tmp/tmp.2gmok3d5BO /tmp/tmp.yhabkAjCqE ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.oXs8oydgaT +++ mktemp ++ local LAST_ERR=/tmp/tmp.EtR3lqJNCf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oXs8oydgaT ++ cat /tmp/tmp.EtR3lqJNCf ++ rm /tmp/tmp.oXs8oydgaT /tmp/tmp.EtR3lqJNCf ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jKuwThJ0HU +++ mktemp ++ local LAST_ERR=/tmp/tmp.hTsgqu4cru ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jKuwThJ0HU ++ cat /tmp/tmp.hTsgqu4cru ++ rm /tmp/tmp.jKuwThJ0HU /tmp/tmp.hTsgqu4cru ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-h5cpp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:47:21.412843 13349 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967638190812000&timeoutSeconds=523&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data smart-update 100500 no-proxy -P3306 + local cluster=smart-update + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UiOCJ0NLnX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JcrNuyDo34 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.UiOCJ0NLnX +++ cat /tmp/tmp.JcrNuyDo34 +++ rm /tmp/tmp.UiOCJ0NLnX /tmp/tmp.JcrNuyDo34 +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.S6HOobpCBK +++ mktemp ++ local LAST_ERR=/tmp/tmp.bgktPc4kKk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.S6HOobpCBK ++ cat /tmp/tmp.bgktPc4kKk ++ rm /tmp/tmp.S6HOobpCBK /tmp/tmp.bgktPc4kKk ++ return 0 + root_pass=root_password ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.77VgF0UNin ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wTtVUwMe9I +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.77VgF0UNin +++ cat /tmp/tmp.wTtVUwMe9I +++ rm /tmp/tmp.77VgF0UNin /tmp/tmp.wTtVUwMe9I +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + proxy=smart-update-haproxy + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h smart-update-pxc-0.smart-update-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xQ2q3y6C8f +++ mktemp ++ local LAST_ERR=/tmp/tmp.nhSjTV5dbh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xQ2q3y6C8f ++ cat /tmp/tmp.nhSjTV5dbh ++ rm /tmp/tmp.xQ2q3y6C8f /tmp/tmp.nhSjTV5dbh ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6666d5df5c-h5cpp ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:47:48.590139 17324 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967665600547000&timeoutSeconds=523&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data smart-update select-1 no-proxy -P3306 + local cluster=smart-update + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RytuB3Nwrm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nqBPIP7XvA +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.RytuB3Nwrm +++ cat /tmp/tmp.nqBPIP7XvA +++ rm /tmp/tmp.RytuB3Nwrm /tmp/tmp.nqBPIP7XvA +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.wNZ3neUBDI +++ mktemp ++ local LAST_ERR=/tmp/tmp.FqJLtakFCK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wNZ3neUBDI ++ cat /tmp/tmp.FqJLtakFCK ++ rm /tmp/tmp.wNZ3neUBDI /tmp/tmp.FqJLtakFCK ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zPiXxTTYpv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XdABjprZch +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.zPiXxTTYpv +++ cat /tmp/tmp.XdABjprZch +++ rm /tmp/tmp.zPiXxTTYpv /tmp/tmp.XdABjprZch +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6CHAU0GbPo +++ mktemp ++ local LAST_ERR=/tmp/tmp.Sd7iSd0slp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6CHAU0GbPo ++ cat /tmp/tmp.Sd7iSd0slp ++ rm /tmp/tmp.6CHAU0GbPo /tmp/tmp.Sd7iSd0slp ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-h5cpp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:48:40.597919 25402 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967718552513000&timeoutSeconds=353&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PAdWkwlIyd +++ mktemp ++ local LAST_ERR=/tmp/tmp.EG7OqpqwEz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PAdWkwlIyd ++ cat /tmp/tmp.EG7OqpqwEz ++ rm /tmp/tmp.PAdWkwlIyd /tmp/tmp.EG7OqpqwEz ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-h5cpp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:48:53.458262 27389 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967730798124000&timeoutSeconds=489&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rm0dQv31VF +++ mktemp ++ local LAST_ERR=/tmp/tmp.FDcHaqz5A8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rm0dQv31VF ++ cat /tmp/tmp.FDcHaqz5A8 ++ rm /tmp/tmp.rm0dQv31VF /tmp/tmp.FDcHaqz5A8 ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ echo pxc-client-6666d5df5c-h5cpp ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:49:08.161771 29454 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967744374279000&timeoutSeconds=404&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + kubectl exec smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + grep -E -o 'early-plugin-load=keyring_\w+.so' + return 1 ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.wBe82lTyVL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.M0e4BrmQnX +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.wBe82lTyVL +++ cat /tmp/tmp.M0e4BrmQnX +++ rm /tmp/tmp.wBe82lTyVL /tmp/tmp.M0e4BrmQnX +++ return 0 ++ client_pod=pxc-client-6666d5df5c-h5cpp ++ wait_pod pxc-client-6666d5df5c-h5cpp ++ local pod=pxc-client-6666d5df5c-h5cpp ++ local max_retry=480 ++ local ns= +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ grep -E '^(pxc|proxysql)$' +++ echo pxc-client-6666d5df5c-h5cpp ++ local container= ++ set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:49:27.348899 32350 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967763931399000&timeoutSeconds=563&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.F8cWA51cCo ++ mktemp + local LAST_ERR=/tmp/tmp.TbguOFe7TV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.F8cWA51cCo perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.TbguOFe7TV + rm /tmp/tmp.F8cWA51cCo /tmp/tmp.TbguOFe7TV + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update.......... smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9t2tt2QFPN +++ mktemp ++ local LAST_ERR=/tmp/tmp.waQOHgq7PL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9t2tt2QFPN ++ cat /tmp/tmp.waQOHgq7PL ++ rm /tmp/tmp.9t2tt2QFPN /tmp/tmp.waQOHgq7PL ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sWuVy3Zewh +++ mktemp ++ local LAST_ERR=/tmp/tmp.vegAUkN3IV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sWuVy3Zewh ++ cat /tmp/tmp.vegAUkN3IV ++ rm /tmp/tmp.sWuVy3Zewh /tmp/tmp.vegAUkN3IV ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.FhiR85IBam ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.mDJFBaxUhi +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.FhiR85IBam +++++ cat /tmp/tmp.mDJFBaxUhi +++++ rm /tmp/tmp.FhiR85IBam /tmp/tmp.mDJFBaxUhi +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jD0b8LJ86r +++ mktemp ++ local LAST_ERR=/tmp/tmp.7fCxlyi3B0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jD0b8LJ86r ++ cat /tmp/tmp.7fCxlyi3B0 ++ rm /tmp/tmp.jD0b8LJ86r /tmp/tmp.7fCxlyi3B0 ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.O8C9VumoRz +++ mktemp ++ local LAST_ERR=/tmp/tmp.PqgCeucaK6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.O8C9VumoRz ++ cat /tmp/tmp.PqgCeucaK6 ++ rm /tmp/tmp.O8C9VumoRz /tmp/tmp.PqgCeucaK6 ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vXiaU9dvrM +++ mktemp ++ local LAST_ERR=/tmp/tmp.kxJHIHtxG5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vXiaU9dvrM ++ cat /tmp/tmp.kxJHIHtxG5 ++ rm /tmp/tmp.vXiaU9dvrM /tmp/tmp.kxJHIHtxG5 ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.BtkzZSVUsj ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.n8WV7BpdtS +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.BtkzZSVUsj +++++ cat /tmp/tmp.n8WV7BpdtS +++++ rm /tmp/tmp.BtkzZSVUsj /tmp/tmp.n8WV7BpdtS +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZMDkppDFX6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.cjwK76C9CW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZMDkppDFX6 ++ cat /tmp/tmp.cjwK76C9CW ++ rm /tmp/tmp.ZMDkppDFX6 /tmp/tmp.cjwK76C9CW ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.85rjvBfAyE +++ mktemp ++ local LAST_ERR=/tmp/tmp.w5gcxU8I3D ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.85rjvBfAyE ++ cat /tmp/tmp.w5gcxU8I3D ++ rm /tmp/tmp.85rjvBfAyE /tmp/tmp.w5gcxU8I3D ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gsClEgTlyH +++ mktemp ++ local LAST_ERR=/tmp/tmp.sShTgAUpPI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gsClEgTlyH ++ cat /tmp/tmp.sShTgAUpPI ++ rm /tmp/tmp.gsClEgTlyH /tmp/tmp.sShTgAUpPI ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.34cyhkE7DV ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.1NjaLE6iy5 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.34cyhkE7DV +++++ cat /tmp/tmp.1NjaLE6iy5 +++++ rm /tmp/tmp.34cyhkE7DV /tmp/tmp.1NjaLE6iy5 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0ODg55kiss +++ mktemp ++ local LAST_ERR=/tmp/tmp.QSqLXyz6wI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0ODg55kiss ++ cat /tmp/tmp.QSqLXyz6wI ++ rm /tmp/tmp.0ODg55kiss /tmp/tmp.QSqLXyz6wI ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CCvUHjHrS8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.1kAYIiHmbM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CCvUHjHrS8 ++ cat /tmp/tmp.1kAYIiHmbM ++ rm /tmp/tmp.CCvUHjHrS8 /tmp/tmp.1kAYIiHmbM ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PlHtW7kNmS +++ mktemp ++ local LAST_ERR=/tmp/tmp.aXOOxXyZUt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PlHtW7kNmS ++ cat /tmp/tmp.aXOOxXyZUt ++ rm /tmp/tmp.PlHtW7kNmS /tmp/tmp.aXOOxXyZUt ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.49PfrjxUH3 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.rdbiAWrzeL +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.49PfrjxUH3 +++++ cat /tmp/tmp.rdbiAWrzeL +++++ rm /tmp/tmp.49PfrjxUH3 /tmp/tmp.rdbiAWrzeL +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.29DWB6Sloq +++ mktemp ++ local LAST_ERR=/tmp/tmp.G4wz0ntBuw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.29DWB6Sloq ++ cat /tmp/tmp.G4wz0ntBuw ++ rm /tmp/tmp.29DWB6Sloq /tmp/tmp.G4wz0ntBuw ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ufero9HZCt +++ mktemp ++ local LAST_ERR=/tmp/tmp.KqLHTNyLuy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ufero9HZCt ++ cat /tmp/tmp.KqLHTNyLuy ++ rm /tmp/tmp.Ufero9HZCt /tmp/tmp.KqLHTNyLuy ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.efL0bwbnW2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.mAHBqyGVoX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.efL0bwbnW2 ++ cat /tmp/tmp.mAHBqyGVoX ++ rm /tmp/tmp.efL0bwbnW2 /tmp/tmp.mAHBqyGVoX ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.8dpzW1IqdQ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ewhKx2kvwJ +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.8dpzW1IqdQ +++++ cat /tmp/tmp.ewhKx2kvwJ +++++ rm /tmp/tmp.8dpzW1IqdQ /tmp/tmp.ewhKx2kvwJ +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AoovKXBsOR +++ mktemp ++ local LAST_ERR=/tmp/tmp.D4GI5maJIq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AoovKXBsOR ++ cat /tmp/tmp.D4GI5maJIq ++ rm /tmp/tmp.AoovKXBsOR /tmp/tmp.D4GI5maJIq ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YMqoTg4tHs +++ mktemp ++ local LAST_ERR=/tmp/tmp.5vhhnJnZxw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YMqoTg4tHs ++ cat /tmp/tmp.5vhhnJnZxw ++ rm /tmp/tmp.YMqoTg4tHs /tmp/tmp.5vhhnJnZxw ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XQzLOXgZm4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.UQZdRk6l6P ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XQzLOXgZm4 ++ cat /tmp/tmp.UQZdRk6l6P ++ rm /tmp/tmp.XQzLOXgZm4 /tmp/tmp.UQZdRk6l6P ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.spHhLdfmRt ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.2p5EnG2wXg +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.spHhLdfmRt +++++ cat /tmp/tmp.2p5EnG2wXg +++++ rm /tmp/tmp.spHhLdfmRt /tmp/tmp.2p5EnG2wXg +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IhHSp96jGV +++ mktemp ++ local LAST_ERR=/tmp/tmp.3yQOqry2EY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IhHSp96jGV ++ cat /tmp/tmp.3yQOqry2EY ++ rm /tmp/tmp.IhHSp96jGV /tmp/tmp.3yQOqry2EY ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GrcXm5iLGQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.xBX5bK3hnG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GrcXm5iLGQ ++ cat /tmp/tmp.xBX5bK3hnG ++ rm /tmp/tmp.GrcXm5iLGQ /tmp/tmp.xBX5bK3hnG ++ return 0 + state=initializing ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rwKSuvOOSZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.mZmxImKQL0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rwKSuvOOSZ ++ cat /tmp/tmp.mZmxImKQL0 ++ rm /tmp/tmp.rwKSuvOOSZ /tmp/tmp.mZmxImKQL0 ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.cpJ84S85Po ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.5bbKn0EkN2 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.cpJ84S85Po +++++ cat /tmp/tmp.5bbKn0EkN2 +++++ rm /tmp/tmp.cpJ84S85Po /tmp/tmp.5bbKn0EkN2 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z5YZr74mIn +++ mktemp ++ local LAST_ERR=/tmp/tmp.oQHFgHQ7dn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Z5YZr74mIn ++ cat /tmp/tmp.oQHFgHQ7dn ++ rm /tmp/tmp.Z5YZr74mIn /tmp/tmp.oQHFgHQ7dn ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hOhMTncyAp +++ mktemp ++ local LAST_ERR=/tmp/tmp.6SDq7d48xH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hOhMTncyAp ++ cat /tmp/tmp.6SDq7d48xH ++ rm /tmp/tmp.hOhMTncyAp /tmp/tmp.6SDq7d48xH ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AaXoeqrCcv +++ mktemp ++ local LAST_ERR=/tmp/tmp.FrFJj7ZdaM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AaXoeqrCcv ++ cat /tmp/tmp.FrFJj7ZdaM ++ rm /tmp/tmp.AaXoeqrCcv /tmp/tmp.FrFJj7ZdaM ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.sOmGWcxzdF ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Ma0aPDahCK +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.sOmGWcxzdF +++++ cat /tmp/tmp.Ma0aPDahCK +++++ rm /tmp/tmp.sOmGWcxzdF /tmp/tmp.Ma0aPDahCK +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CMKNvwyfd4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.lhoKOQOwnm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CMKNvwyfd4 ++ cat /tmp/tmp.lhoKOQOwnm ++ rm /tmp/tmp.CMKNvwyfd4 /tmp/tmp.lhoKOQOwnm ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WYjTJHZVk4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FXFREwrhAw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WYjTJHZVk4 ++ cat /tmp/tmp.FXFREwrhAw ++ rm /tmp/tmp.WYjTJHZVk4 /tmp/tmp.FXFREwrhAw ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-h5cpp + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:52:58.685423 32291 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967974748083000&timeoutSeconds=589&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PMB7ZYgYYf +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z82LboFFjw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PMB7ZYgYYf ++ cat /tmp/tmp.Z82LboFFjw ++ rm /tmp/tmp.PMB7ZYgYYf /tmp/tmp.Z82LboFFjw ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6666d5df5c-h5cpp + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:53:16.024614 2626 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782967992616158000&timeoutSeconds=379&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.75OQJD9Gzy +++ mktemp ++ local LAST_ERR=/tmp/tmp.tkkTry0RPu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.75OQJD9Gzy ++ cat /tmp/tmp.tkkTry0RPu ++ rm /tmp/tmp.75OQJD9Gzy /tmp/tmp.tkkTry0RPu ++ return 0 + client_pod=pxc-client-6666d5df5c-h5cpp + wait_pod pxc-client-6666d5df5c-h5cpp + local pod=pxc-client-6666d5df5c-h5cpp + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6666d5df5c-h5cpp ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6666d5df5c-h5cpp condition met E0702 04:53:32.640741 5129 reflector.go:227] "Failed to watch" err="Get \"https://136.65.113.92/api/v1/namespaces/smart-update1-19768/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-6666d5df5c-h5cpp&resourceVersion=1782968009230922000&timeoutSeconds=498&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-6666d5df5c-h5cpp to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Cj53gF286S/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Cj53gF286S/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Cj53gF286S/select-1.sql + kubectl_bin delete -f /tmp/tmp.Cj53gF286S/smart-update-version-service-reachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.umKYJTCFnC ++ mktemp + local LAST_ERR=/tmp/tmp.9VD8w0ZILx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.Cj53gF286S/smart-update-version-service-reachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.umKYJTCFnC perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update1-19768 namespace + cat /tmp/tmp.9VD8w0ZILx + rm /tmp/tmp.umKYJTCFnC /tmp/tmp.9VD8w0ZILx + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.Gr0rE0BlXH ++ mktemp + local LAST_ERR=/tmp/tmp.pnxrhzwTH0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Gr0rE0BlXH persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update1-19768 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update1-19768 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update1-19768 namespace + cat /tmp/tmp.pnxrhzwTH0 + rm /tmp/tmp.Gr0rE0BlXH /tmp/tmp.pnxrhzwTH0 + return 0 + desc cleanup + set +o xtrace ----------------------------------------------------------------------------------- cleanup ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.RDs8b2EYvZ ++ mktemp + local LAST_ERR=/tmp/tmp.GYkvOTdrcg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2538/e2e-tests/smart-update1/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RDs8b2EYvZ deployment.apps "version-service" deleted from smart-update1-19768 namespace service "version-service" deleted from smart-update1-19768 namespace + cat /tmp/tmp.GYkvOTdrcg + rm /tmp/tmp.RDs8b2EYvZ /tmp/tmp.GYkvOTdrcg + return 0 + destroy smart-update1-19768 + local namespace=smart-update1-19768 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v 'get backup status: Job.batch' + grep -v 'the object has been modified' + tee /tmp/tmp.Cj53gF286S/operator.log + grep -v level=info + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + sort -u ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.jttbKpfhA8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.1jMIuzWdIF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jttbKpfhA8 ++ cat /tmp/tmp.1jMIuzWdIF ++ rm /tmp/tmp.jttbKpfhA8 /tmp/tmp.1jMIuzWdIF ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-59848dbdcc-crj4g ++ mktemp + local LAST_OUT=/tmp/tmp.ASRvK8pUaI ++ mktemp + local LAST_ERR=/tmp/tmp.wfQah0sPsO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-59848dbdcc-crj4g + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ASRvK8pUaI + cat /tmp/tmp.wfQah0sPsO + rm /tmp/tmp.ASRvK8pUaI /tmp/tmp.wfQah0sPsO + return 0 } }, }, { }, }, { }, }, }, ""), { }, - }, - { - }, + }, ... // 16 identical fields ... // 16 identical fields 2026-07-02T04:18:00.044Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-07-02T04:18:00.044Z INFO setup Manager starting up {"gitCommit": "70c04858efdd60ae80fd1cf4dcb3edc388dbd027", "gitBranch": "PR-2538-70c04858", "buildTime": "2026-07-02T01:14:54Z", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} 2026-07-02T04:18:00.044Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.33.12-gke.1059000"} 2026-07-02T04:18:00.047Z INFO setup Registering Components. 2026-07-02T04:18:00.937Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-07-02T04:18:00.937Z INFO setup Starting the Cmd. 2026-07-02T04:18:00.938Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-07-02T04:18:00.938Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-07-02T04:18:00.938Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-07-02T04:18:00.938Z INFO controller-runtime.metrics Starting metrics server 2026-07-02T04:18:00.938Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-07-02T04:18:00.938Z INFO controller-runtime.webhook Starting webhook server 2026-07-02T04:18:00.938Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-07-02T04:18:01.038Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-07-02T04:18:01.068Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-07-02T04:18:01.069Z DEBUG events percona-xtradb-cluster-operator-59848dbdcc-crj4g_3d22cd66-5d90-4f4a-a9bb-58c91b15fe60 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"70fb85b5-12cd-4fdd-9b9f-a805b84152fb","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1782965881060511009"}, "reason": "LeaderElection"} 2026-07-02T04:18:01.069Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-07-02T04:18:01.069Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-07-02T04:18:01.069Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-07-02T04:18:01.069Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-07-02T04:18:01.169Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-07-02T04:18:01.170Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-07-02T04:18:01.271Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-07-02T04:18:01.271Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-07-02T04:18:01.271Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-07-02T04:18:01.271Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-07-02T04:19:36.029Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "05821452-042c-48bf-bc88-ae9584d890e8", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:19:36.147Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "05821452-042c-48bf-bc88-ae9584d890e8", "object": "smart-update-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:19:36.194Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "05821452-042c-48bf-bc88-ae9584d890e8", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:19:36.236Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "05821452-042c-48bf-bc88-ae9584d890e8", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:19:36.255Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "05821452-042c-48bf-bc88-ae9584d890e8", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:19:36.326Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "05821452-042c-48bf-bc88-ae9584d890e8", "object": "smart-update-proxysql", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:19:36.424Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "05821452-042c-48bf-bc88-ae9584d890e8", "object": "smart-update-proxysql-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:19:37.304Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "19970d6d-616e-4d91-9472-10292fdad65a", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:19:37.321Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "19970d6d-616e-4d91-9472-10292fdad65a", "object": "smart-update-proxysql", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:20:24.531Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "44b9a238-5a9b-4b37-b52f-ba407cc51e30", "user": "operator"} 2026-07-02T04:20:24.558Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "44b9a238-5a9b-4b37-b52f-ba407cc51e30", "user": "monitor"} 2026-07-02T04:20:24.608Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "44b9a238-5a9b-4b37-b52f-ba407cc51e30"} 2026-07-02T04:20:24.637Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "44b9a238-5a9b-4b37-b52f-ba407cc51e30"} 2026-07-02T04:20:24.684Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "44b9a238-5a9b-4b37-b52f-ba407cc51e30", "user": "xtrabackup"} 2026-07-02T04:20:24.728Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "44b9a238-5a9b-4b37-b52f-ba407cc51e30"} 2026-07-02T04:20:24.756Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "44b9a238-5a9b-4b37-b52f-ba407cc51e30", "user": "replication"} 2026-07-02T04:20:25.766Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "44b9a238-5a9b-4b37-b52f-ba407cc51e30", "err": "get primary pxc pod: not found"} 2026-07-02T04:20:31.885Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fe27813f-2188-417b-9556-93f0b20efdca", "err": "get primary pxc pod: not found"} 2026-07-02T04:20:38.434Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "57407e32-3744-4e4f-857e-702cf5df7d2f", "err": "get primary pxc pod: not found"} 2026-07-02T04:22:39.324Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "b67963b6-2293-4b3a-b93d-038632e674be", "user": "root"} 2026-07-02T04:22:40.391Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "b67963b6-2293-4b3a-b93d-038632e674be"} 2026-07-02T04:22:40.428Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "b67963b6-2293-4b3a-b93d-038632e674be", "new version": "8.0.41-32.1"} 2026-07-02T04:22:42.052Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "05328e61-7910-4257-8e8a-9d9365bf9732"} 2026-07-02T04:22:48.504Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "780bc86e-d855-4ce4-8f96-afd26d0af25d"} 2026-07-02T04:22:54.518Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "aa8524bd-f21e-458a-b83b-6f591002a683"} 2026-07-02T04:23:00.787Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "ea5bf111-944b-48b7-a725-8894ef0f89d8"} 2026-07-02T04:23:07.009Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "f7ee43c2-1c56-4df3-b527-06e72a16a631"} 2026-07-02T04:23:13.513Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "ad019288-de08-4091-822c-851e3304e561"} 2026-07-02T04:23:19.614Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fa9ad35c-c985-4935-b9f1-52d603272892"} 2026-07-02T04:23:25.787Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "6f7667ba-ca45-4f89-aa89-210fb8a03782"} 2026-07-02T04:23:32.976Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "729c9740-aff9-43fa-b22d-725574f198a5"} 2026-07-02T04:23:39.225Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "9f976dff-606b-457d-9c0a-7ec17a6ad666"} 2026-07-02T04:23:45.522Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "e0984fbf-9e80-41c6-bdf2-d76fc11b9f22"} 2026-07-02T04:23:51.831Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "81aacf5b-dce7-4cb4-9779-9e6416e23921"} 2026-07-02T04:23:57.954Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "247db909-34ea-4528-b01a-3eebabb6d598"} 2026-07-02T04:24:04.382Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "74c6dc18-e9d3-4649-a69d-789f647c39de"} 2026-07-02T04:24:10.304Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "09e8bcd1-91e5-4def-ab34-c5d5fe54c6ee"} 2026-07-02T04:24:16.872Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "d5013593-45d8-436d-b7bb-e616c21a8af1"} 2026-07-02T04:24:23.231Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "06470ce3-11ef-4019-bc13-87d072a258fd"} 2026-07-02T04:24:29.503Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "d3352d01-b5ce-4ca0-9847-1d57e8417f96"} 2026-07-02T04:24:35.935Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "30f86b4c-0bf9-4772-be9e-5e6c3a124199"} 2026-07-02T04:24:42.023Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "f668b33b-b83c-4cd9-a1ff-7a2ed2d6ef46"} 2026-07-02T04:24:48.530Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "0b8c69e7-d155-4c5c-90b8-c91c006f0636"} 2026-07-02T04:24:54.524Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "a6090919-84d6-49eb-ae68-73ef44b79324"} 2026-07-02T04:25:00.597Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "5cc5c277-cb6e-418b-a988-a28d31f97a2c"} 2026-07-02T04:25:02.475Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T04:25:02.534Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T04:25:03.684Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647"} 2026-07-02T04:25:03.690Z INFO apply changes to secondary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2"} 2026-07-02T04:25:03.690Z INFO primary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-0.smart-update-pxc.smart-update1-19768.svc.cluster.local"} 2026-07-02T04:25:03.908Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647"} 2026-07-02T04:25:04.348Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2", "currentRevision": "smart-update-pxc-79c7775b5b", "targetRevision": "smart-update-pxc-779b69f9b5"} 2026-07-02T04:25:14.446Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2", "currentRevision": "smart-update-pxc-79c7775b5b", "targetRevision": "smart-update-pxc-779b69f9b5"} 2026-07-02T04:25:24.428Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2", "phase": "Pending"} 2026-07-02T04:25:24.428Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2", "reason": "PodInitializing"} 2026-07-02T04:25:34.418Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2", "phase": "Pending"} 2026-07-02T04:25:34.418Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2", "reason": "PodInitializing"} 2026-07-02T04:26:14.447Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2"} 2026-07-02T04:26:14.464Z INFO waiting for pod to be present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2", "host": "smart-update-pxc-2.smart-update-pxc.smart-update1-19768", "hostgroup": 10} 2026-07-02T04:26:24.466Z INFO waiting for pod to be present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2", "host": "smart-update-pxc-2.smart-update-pxc.smart-update1-19768", "hostgroup": 10} 2026-07-02T04:26:34.467Z INFO waiting for pod to be present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2", "host": "smart-update-pxc-2.smart-update-pxc.smart-update1-19768", "hostgroup": 10} 2026-07-02T04:26:44.467Z INFO waiting for pod to be present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2", "host": "smart-update-pxc-2.smart-update-pxc.smart-update1-19768", "hostgroup": 10} 2026-07-02T04:26:54.466Z INFO pod present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2", "host": "smart-update-pxc-2.smart-update-pxc.smart-update1-19768", "hostgroup": 10} 2026-07-02T04:26:54.471Z INFO apply changes to secondary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-1"} 2026-07-02T04:26:54.471Z INFO pod is online {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-2"} 2026-07-02T04:26:54.672Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-1", "currentRevision": "smart-update-pxc-79c7775b5b", "targetRevision": "smart-update-pxc-779b69f9b5"} 2026-07-02T04:27:04.788Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-1", "currentRevision": "smart-update-pxc-79c7775b5b", "targetRevision": "smart-update-pxc-779b69f9b5"} 2026-07-02T04:27:14.744Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-1", "phase": "Pending"} 2026-07-02T04:27:14.744Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-1", "reason": "PodInitializing"} 2026-07-02T04:27:54.764Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-1"} 2026-07-02T04:27:54.782Z INFO pod present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-1", "host": "smart-update-pxc-1.smart-update-pxc.smart-update1-19768", "hostgroup": 10} 2026-07-02T04:27:54.786Z INFO apply changes to primary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-0"} 2026-07-02T04:27:54.786Z INFO pod is online {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-1"} 2026-07-02T04:27:54.929Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-0", "currentRevision": "smart-update-pxc-79c7775b5b", "targetRevision": "smart-update-pxc-779b69f9b5"} 2026-07-02T04:28:05.020Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-0", "currentRevision": "smart-update-pxc-79c7775b5b", "targetRevision": "smart-update-pxc-779b69f9b5"} 2026-07-02T04:28:14.997Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-0", "phase": "Pending"} 2026-07-02T04:28:14.997Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-0", "reason": "PodInitializing"} 2026-07-02T04:28:55.024Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-0"} 2026-07-02T04:28:55.041Z INFO pod present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-0", "host": "smart-update-pxc-0.smart-update-pxc.smart-update1-19768", "hostgroup": 11} 2026-07-02T04:28:55.045Z INFO pod is online {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "pod": "smart-update-pxc-0"} 2026-07-02T04:28:55.045Z INFO smart update finished {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647"} 2026-07-02T04:28:55.110Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "defafde4-3abe-4c38-b375-903ab9118647", "new version": "8.0.43-34.1"} 2026-07-02T04:28:57.104Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "f3c640bd-7b1a-4508-9de3-d6c79963e3ae"} 2026-07-02T04:29:04.042Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "e9ddcbd6-0b64-404a-bdd5-2a306b16cc82"} 2026-07-02T04:29:10.081Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "f01ba456-dd8f-4132-8c65-cbdfa27ee235"} 2026-07-02T04:29:16.645Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "28ad0608-445a-4e05-a83c-d1dac7877a13"} 2026-07-02T04:29:22.989Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "b749474a-c1bc-48f9-9026-61e45da15bdc"} 2026-07-02T04:29:29.028Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "8ef27359-03d6-44fb-ba23-446241b90c9b"} 2026-07-02T04:29:35.350Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "0eb25a05-deed-42be-9298-e1da45de467f"} 2026-07-02T04:29:41.489Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "51b16317-b2ab-47a8-9935-4e9440ee4014"} 2026-07-02T04:29:48.105Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "acf4e709-0508-453b-b44d-c5e456ddf100"} 2026-07-02T04:29:54.185Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "1ca45ef5-432f-4b13-bcee-7a945026ac78"} 2026-07-02T04:30:35.804Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "3b6be230-769e-460c-9861-8ac9aae27e1f", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2026-07-02T04:30:55.680Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "f1d30d28-dc55-442b-a7e8-d6bb2da079fd", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:30:55.693Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "f1d30d28-dc55-442b-a7e8-d6bb2da079fd", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:30:56.246Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "f1d30d28-dc55-442b-a7e8-d6bb2da079fd", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:30:56.340Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "f56aa2b3-e381-438b-b9a5-438f6f07926a", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:30:56.380Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "f56aa2b3-e381-438b-b9a5-438f6f07926a", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:30:56.424Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "f56aa2b3-e381-438b-b9a5-438f6f07926a", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:30:56.482Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "f56aa2b3-e381-438b-b9a5-438f6f07926a", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:30:56.554Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "f56aa2b3-e381-438b-b9a5-438f6f07926a", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:30:56.682Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "f56aa2b3-e381-438b-b9a5-438f6f07926a", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:30:57.597Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "51d35d2e-0711-47ba-9215-6e69455421ae", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:30:57.616Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "51d35d2e-0711-47ba-9215-6e69455421ae", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:34:57.698Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "cedcbdd7-9367-447c-bc54-3c6de07c5812", "new version": "8.0.41-32.1"} 2026-07-02T04:37:36.325Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T04:37:36.379Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T04:37:37.468Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346"} 2026-07-02T04:37:37.476Z INFO apply changes to secondary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-2"} 2026-07-02T04:37:37.476Z INFO primary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-0.smart-update-pxc.smart-update1-19768"} 2026-07-02T04:37:38.150Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-2", "currentRevision": "smart-update-pxc-544f97b756", "targetRevision": "smart-update-pxc-759f7f6db9"} 2026-07-02T04:37:48.324Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-2", "currentRevision": "smart-update-pxc-544f97b756", "targetRevision": "smart-update-pxc-759f7f6db9"} 2026-07-02T04:37:58.240Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-2", "phase": "Pending"} 2026-07-02T04:37:58.240Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-2", "reason": "PodInitializing"} 2026-07-02T04:38:38.281Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-2"} 2026-07-02T04:38:38.294Z INFO apply changes to secondary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-1"} 2026-07-02T04:38:38.497Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-1", "currentRevision": "smart-update-pxc-544f97b756", "targetRevision": "smart-update-pxc-759f7f6db9"} 2026-07-02T04:38:48.596Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-1", "currentRevision": "smart-update-pxc-544f97b756", "targetRevision": "smart-update-pxc-759f7f6db9"} 2026-07-02T04:38:58.627Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-1", "phase": "Pending"} 2026-07-02T04:38:58.627Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-1", "reason": "PodInitializing"} 2026-07-02T04:39:38.594Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-1"} 2026-07-02T04:39:38.607Z INFO apply changes to primary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-0"} 2026-07-02T04:39:38.770Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-0", "currentRevision": "smart-update-pxc-544f97b756", "targetRevision": "smart-update-pxc-759f7f6db9"} 2026-07-02T04:39:48.871Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-0", "currentRevision": "smart-update-pxc-544f97b756", "targetRevision": "smart-update-pxc-759f7f6db9"} 2026-07-02T04:39:58.843Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-0", "phase": "Pending"} 2026-07-02T04:39:58.843Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-0", "reason": "PodInitializing"} 2026-07-02T04:40:38.878Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "pod": "smart-update-pxc-0"} 2026-07-02T04:40:38.892Z INFO smart update finished {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346"} 2026-07-02T04:40:38.980Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "fabddc0f-6eca-4755-9ce7-4ecded0b2346", "new version": "8.0.43-34.1"} 2026-07-02T04:41:51.912Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "322b4717-8a7f-4a9a-b182-52376d58ab62", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2026-07-02T04:42:13.556Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "271c93d5-76b3-4a34-be2c-d2c977acce47", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:42:13.573Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "271c93d5-76b3-4a34-be2c-d2c977acce47", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T04:42:14.132Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "271c93d5-76b3-4a34-be2c-d2c977acce47", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T04:42:14.231Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "a3d3152d-cc3e-44aa-8be6-6d0555126528", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:42:14.265Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "a3d3152d-cc3e-44aa-8be6-6d0555126528", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:42:14.306Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "a3d3152d-cc3e-44aa-8be6-6d0555126528", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:42:14.354Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "a3d3152d-cc3e-44aa-8be6-6d0555126528", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:42:14.396Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "a3d3152d-cc3e-44aa-8be6-6d0555126528", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:42:14.552Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "a3d3152d-cc3e-44aa-8be6-6d0555126528", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:42:15.343Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "4b41b835-a94f-4308-9cb1-0414dad5260f", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:42:15.358Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "4b41b835-a94f-4308-9cb1-0414dad5260f", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:46:12.581Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "b87a6bef-8237-4ae6-9fd9-a4ba002196c1", "new version": "8.0.41-32.1"} 2026-07-02T04:49:37.672Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T04:49:37.757Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T04:49:38.922Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56"} 2026-07-02T04:49:38.931Z INFO apply changes to secondary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-2"} 2026-07-02T04:49:38.931Z INFO primary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-0.smart-update-pxc.smart-update1-19768"} 2026-07-02T04:49:40.113Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-2", "currentRevision": "smart-update-pxc-867d655857", "targetRevision": "smart-update-pxc-6479fc9d88"} 2026-07-02T04:49:50.217Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-2", "currentRevision": "smart-update-pxc-867d655857", "targetRevision": "smart-update-pxc-6479fc9d88"} 2026-07-02T04:50:30.221Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-2"} 2026-07-02T04:50:30.233Z INFO apply changes to secondary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-1"} 2026-07-02T04:50:30.443Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-1", "currentRevision": "smart-update-pxc-867d655857", "targetRevision": "smart-update-pxc-6479fc9d88"} 2026-07-02T04:50:40.535Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-1", "currentRevision": "smart-update-pxc-867d655857", "targetRevision": "smart-update-pxc-6479fc9d88"} 2026-07-02T04:50:50.513Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-1", "phase": "Pending"} 2026-07-02T04:50:50.513Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-1", "reason": "PodInitializing"} 2026-07-02T04:51:30.538Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-1"} 2026-07-02T04:51:30.549Z INFO apply changes to primary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-0"} 2026-07-02T04:51:30.739Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-0", "currentRevision": "smart-update-pxc-867d655857", "targetRevision": "smart-update-pxc-6479fc9d88"} 2026-07-02T04:51:40.835Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-0", "currentRevision": "smart-update-pxc-867d655857", "targetRevision": "smart-update-pxc-6479fc9d88"} 2026-07-02T04:51:50.830Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-0", "phase": "Pending"} 2026-07-02T04:51:50.830Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-0", "reason": "PodInitializing"} 2026-07-02T04:52:30.835Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "pod": "smart-update-pxc-0"} 2026-07-02T04:52:30.845Z INFO smart update finished {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56"} 2026-07-02T04:52:30.907Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "7ee76150-62bd-46f0-967f-04c6a4726c56", "new version": "8.0.43-34.1"} 2026-07-02T04:54:24.549Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-19768"}, "namespace": "smart-update1-19768", "name": "smart-update", "reconcileID": "3bb5c308-c8a2-486e-b7cb-f019d38e6269", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} ... // 22 identical fields ... // 2 identical fields ... // 3 identical fields ... // 3 identical fields ... // 3 identical fields ... // 4 identical fields ... // 5 identical fields ... // 6 identical fields ... // 7 identical fields - ":8.0.41-32.1", ... // 9 identical fields AccessModes: nil, ActiveDeadlineSeconds: nil, Affinity: &{PodAntiAffinity: &{RequiredDuringSchedulingIgnoredDuringExecution: {{LabelSelector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "smart-update", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, TopologyKey: "kubernetes.io/hostname"}}}}, - Annotations: map[string]string{ + Annotations: map[string]string{ - APIVersion: "apps/v1", - APIVersion: "apps/v1", Args: {"mysqld"}, AutomountServiceAccountToken: nil, + AvailableReplicas: 0, - AvailableReplicas: 3, AWSElasticBlockStore: nil, AzureFile: nil, Capacity: nil, - CollisionCount: &0, + CollisionCount: nil, Command: {"/var/lib/mysql/pxc-entrypoint.sh"}, Conditions: nil, ConfigMap: &v1.ConfigMapVolumeSource{ ContainerPort: 3306, ContainerPort: 33060, ContainerPort: 33062, ContainerPort: 4444, ContainerPort: 4567, ContainerPort: 4568, Containers: []v1.Container{ + CreationTimestamp: v1.Time{}, - CreationTimestamp: v1.Time{Time: s"2026-07-02 04:19:36 +0000 UTC"}, - CreationTimestamp: v1.Time{Time: s"2026-07-02 04:30:56 +0000 UTC"}, - CreationTimestamp: v1.Time{Time: s"2026-07-02 04:42:14 +0000 UTC"}, + CurrentReplicas: 0, - CurrentReplicas: 3, + CurrentRevision: "", - CurrentRevision: "smart-update-pxc-544f97b756", - CurrentRevision: "smart-update-pxc-79c7775b5b", - CurrentRevision: "smart-update-pxc-867d655857", DataSource: nil, DataSourceRef: nil, - DefaultMode: &420, - DefaultMode: &420, + DefaultMode: nil, + DefaultMode: nil, DeletionGracePeriodSeconds: nil, DeletionTimestamp: nil, + DeprecatedServiceAccount: "", - DeprecatedServiceAccount: "default", + DNSPolicy: "", - DNSPolicy: "ClusterFirst", EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "smart-update-env-vars-pxc"}, Optional: &true}}}, Env: {{Name: "PXC_SERVICE", Value: "smart-update-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-smart-update"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-smart-update"}, Key: "xtrabackup"}}}, ...}, EphemeralContainers: nil, FailureThreshold: 3, FC: nil, - FieldsType: "FieldsV1", - FieldsType: "FieldsV1", - FieldsV1: s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`..., - FieldsV1: s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`..., Finalizers: nil, + Generation: 0, - Generation: 1, github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1 GitRepo: nil, /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:272 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:509 HostAliases: nil, HostIP: "", HostPort: 0, ImagePullPolicy: "Always", Image: strings.Join({ InitContainers: []v1.Container{ InitialDelaySeconds: 300, ISCSI: nil, Items: nil, Items: nil, + "lab", Labels: nil, Lifecycle: nil, LivenessProbe: &v1.Probe{ LocalObjectReference: {Name: "auto-smart-update-pxc"}, LocalObjectReference: {Name: "smart-update-pxc"}, + ManagedFields: nil, - ManagedFields: []v1.ManagedFieldsEntry{ - Manager: "kube-controller-manager", - Manager: "percona-xtradb-cluster-operator", MinReadySeconds: 0, Name: "auto-config", {Name: "bin", VolumeSource: {EmptyDir: &{}}}, Name: "config", Name: "ist", Name: "mysql", Name: "mysql-admin", Name: "mysql-init-file", Name: "mysql-users-secret-file", Name: "mysqlx", Name: "pxc", Namespace: "smart-update1-19768", Name: "ssl", Name: "ssl-internal", Name: "sst", {Name: "tmp", VolumeSource: {EmptyDir: &{}}}, Name: "vault-keyring-secret", Name: "write-set", NFS: nil, NodeName: "", NodeSelector: nil, ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "smart-update", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "a070b11bbf6648a606d7667516364a74", "percona.com/ssl-internal-hash": "c96b30f7c458eff1ed0651fc10329dcc"}}, ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "smart-update", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, ObjectMeta: v1.ObjectMeta{ + ObservedGeneration: 0, - ObservedGeneration: 1, - Operation: "Update", - Operation: "Update", + "-operator:main-pxc8.0", Optional: &false, Optional: &true, Optional: &true, Ordinals: nil, OS: nil, Overhead: nil, OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "smart-update", UID: "637c42a9-35a7-4977-9969-0ce02fa44693", ...}}, OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "smart-update", UID: "8dd11e10-b82f-4482-ab4f-fcc3d7c0a8ec", ...}}, OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "smart-update", UID: "f228a014-e8a6-46de-b16b-fccf6bdeb147", ...}}, "percona", - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzbWFydC11cGRhdGUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifX0sInRlbXBsYXRlIjp7Im1ldGFkYXRhIjp7ImxhYmVscyI6eyJhcHAua3ViZXJuZXRlcy5pby9jb21wb25lbnQiOiJweGMiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6InNtYXJ0LXVwZGF0ZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9LCJhbm5vdGF0aW9ucyI6eyJrdWJlY3RsLmt1YmVybmV0ZXMuaW8vZGVmYXVsdC1jb250YWluZXIiOiJweGMiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiYTA3MGIxMWJiZjY2NDhhNjA2ZDc2Njc1MTYzNjRhNzQiLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6ImM5NmIzMGY3YzQ1OGVmZjFlZDA2NTFmYzEwMzI5ZGNjIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJ0bXAiLCJlbXB0eURpciI6e319LHsibmFtZSI6ImNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoic21hcnQtdXBkYXRlLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtc3NsLWludGVybmFsIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJzc2wiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6InNtYXJ0LXVwZGF0ZS1zc2wiLCJvcHRpb25hbCI6ZmFsc2V9fSx7Im5hbWUiOiJhdXRvLWNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoiYXV0by1zbWFydC11cGRhdGUtcHhjIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJ2YXVsdC1rZXlyaW5nLXNlY3JldCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoic21hcnQtdXBkYXRlLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXlzcWwtaW5pdC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtbXlzcWwtaW5pdCIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoiYmluIiwiZW1wdHlEaXIiOnt9fV0sImluaXRDb250YWluZXJzIjpbeyJuYW1lIjoicHhjLWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTM4LTcwYzA0ODU4IiwiY29tbWFuZCI6WyIvcHhjLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiZGF0YWRpciIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn0seyJuYW1lIjoiYmluIiwibW91bnRQYXRoIjoiL29wdC9wZXJjb25hIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifV0sImNvbnRhaW5lcnMiOlt7Im5hbWUiOiJweGMiLCJpbWFnZSI6InBlcmNvbmEvcGVyY29uYS14dHJhZGItY2x1c3Rlcjo4LjAuNDEtMzIuMSIsImNvbW1hbmQiOlsiL3Zhci9saWIvbXlzcWwvcHhjLWVudHJ5cG9pbnQuc2giXSwiYXJncyI6WyJteXNxbGQiXSwicG9ydHMiOlt7Im5hbWUiOiJteXNxbCIsImNvbnRhaW5lclBvcnQiOjMzMDZ9LHsibmFtZSI6InNzdCIsImNvbnRhaW5lclBvcnQiOjQ0NDR9LHsibmFtZSI6IndyaXRlLXNldCIsImNvbnRhaW5lclBvcnQiOjQ1Njd9LHsibmFtZSI6ImlzdCIsImNvbnRhaW5lclBvcnQiOjQ1Njh9LHsibmFtZSI6Im15c3FsLWFkbWluIiwiY29udGFpbmVyUG9ydCI6MzMwNjJ9LHsibmFtZSI6Im15c3FseCIsImNvbnRhaW5lclBvcnQiOjMzMDYwfV0sImVudkZyb20iOlt7InNlY3JldFJlZiI6eyJuYW1lIjoic21hcnQtdXBkYXRlLWVudi12YXJzLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX1dLCJlbnYiOlt7Im5hbWUiOiJQWENfU0VSVklDRSIsInZhbHVlIjoic21hcnQtdXBkYXRlLXB4Yy11bnJlYWR5In0seyJuYW1lIjoiTU9OSVRPUl9IT1NUIiwidmFsdWUiOiIlIn0seyJuYW1lIjoiTVlTUUxfUk9PVF9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6InJvb3QifX19LHsibmFtZSI6IlhUUkFCQUNLVVBfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJ4dHJhYmFja3VwIn19fSx7Im5hbWUiOiJNT05JVE9SX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5IjoibW9uaXRvciJ9fX0seyJuYW1lIjoiQ0xVU1RFUl9IQVNIIiwidmFsdWUiOiIyNjg0MTExIn0seyJuYW1lIjoiT1BFUkFUT1JfQURNSU5fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJvcGVyYXRvciJ9fX0seyJuYW1lIjoiTElWRU5FU1NfQ0hFQ0tfVElNRU9VVCIsInZhbHVlIjoiNSJ9LHsibmFtZSI6IlJFQURJTkVTU19DSEVDS19USU1FT1VUIiwidmFsdWUiOiIxNSJ9LHsibmFtZSI6IkRFRkFVTFRfQVVUSEVOVElDQVRJT05fUExVR0lOIiwidmFsdWUiOiJjYWNoaW5nX3NoYTJfcGFzc3dvcmQi"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzbWFydC11cGRhdGUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifX0sInRlbXBsYXRlIjp7Im1ldGFkYXRhIjp7ImxhYmVscyI6eyJhcHAua3ViZXJuZXRlcy5pby9jb21wb25lbnQiOiJweGMiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6InNtYXJ0LXVwZGF0ZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9LCJhbm5vdGF0aW9ucyI6eyJrdWJlY3RsLmt1YmVybmV0ZXMuaW8vZGVmYXVsdC1jb250YWluZXIiOiJweGMiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiYTA3MGIxMWJiZjY2NDhhNjA2ZDc2Njc1MTYzNjRhNzQiLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6ImM5NmIzMGY3YzQ1OGVmZjFlZDA2NTFmYzEwMzI5ZGNjIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJ0bXAiLCJlbXB0eURpciI6e319LHsibmFtZSI6ImNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoic21hcnQtdXBkYXRlLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtc3NsLWludGVybmFsIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJzc2wiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6InNtYXJ0LXVwZGF0ZS1zc2wiLCJvcHRpb25hbCI6ZmFsc2V9fSx7Im5hbWUiOiJhdXRvLWNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoiYXV0by1zbWFydC11cGRhdGUtcHhjIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJ2YXVsdC1rZXlyaW5nLXNlY3JldCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoic21hcnQtdXBkYXRlLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXlzcWwtaW5pdC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtbXlzcWwtaW5pdCIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoiYmluIiwiZW1wdHlEaXIiOnt9fV0sImluaXRDb250YWluZXJzIjpbeyJuYW1lIjoicHhjLWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTM4LTcwYzA0ODU4IiwiY29tbWFuZCI6WyIvcHhjLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiZGF0YWRpciIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn0seyJuYW1lIjoiYmluIiwibW91bnRQYXRoIjoiL29wdC9wZXJjb25hIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifV0sImNvbnRhaW5lcnMiOlt7Im5hbWUiOiJweGMiLCJpbWFnZSI6InBlcmNvbmEvcGVyY29uYS14dHJhZGItY2x1c3Rlcjo4LjAuNDEtMzIuMSIsImNvbW1hbmQiOlsiL3Zhci9saWIvbXlzcWwvcHhjLWVudHJ5cG9pbnQuc2giXSwiYXJncyI6WyJteXNxbGQiXSwicG9ydHMiOlt7Im5hbWUiOiJteXNxbCIsImNvbnRhaW5lclBvcnQiOjMzMDZ9LHsibmFtZSI6InNzdCIsImNvbnRhaW5lclBvcnQiOjQ0NDR9LHsibmFtZSI6IndyaXRlLXNldCIsImNvbnRhaW5lclBvcnQiOjQ1Njd9LHsibmFtZSI6ImlzdCIsImNvbnRhaW5lclBvcnQiOjQ1Njh9LHsibmFtZSI6Im15c3FsLWFkbWluIiwiY29udGFpbmVyUG9ydCI6MzMwNjJ9LHsibmFtZSI6Im15c3FseCIsImNvbnRhaW5lclBvcnQiOjMzMDYwfV0sImVudkZyb20iOlt7InNlY3JldFJlZiI6eyJuYW1lIjoic21hcnQtdXBkYXRlLWVudi12YXJzLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX1dLCJlbnYiOlt7Im5hbWUiOiJQWENfU0VSVklDRSIsInZhbHVlIjoic21hcnQtdXBkYXRlLXB4Yy11bnJlYWR5In0seyJuYW1lIjoiTU9OSVRPUl9IT1NUIiwidmFsdWUiOiIlIn0seyJuYW1lIjoiTVlTUUxfUk9PVF9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6InJvb3QifX19LHsibmFtZSI6IlhUUkFCQUNLVVBfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJ4dHJhYmFja3VwIn19fSx7Im5hbWUiOiJNT05JVE9SX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5IjoibW9uaXRvciJ9fX0seyJuYW1lIjoiQ0xVU1RFUl9IQVNIIiwidmFsdWUiOiIzMzI3NDExIn0seyJuYW1lIjoiT1BFUkFUT1JfQURNSU5fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJvcGVyYXRvciJ9fX0seyJuYW1lIjoiTElWRU5FU1NfQ0hFQ0tfVElNRU9VVCIsInZhbHVlIjoiNSJ9LHsibmFtZSI6IlJFQURJTkVTU19DSEVDS19USU1FT1VUIiwidmFsdWUiOiIxNSJ9LHsibmFtZSI6IkRFRkFVTFRfQVVUSEVOVElDQVRJT05fUExVR0lOIiwidmFsdWUiOiJjYWNoaW5nX3NoYTJfcGFzc3dvcmQi"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzbWFydC11cGRhdGUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifX0sInRlbXBsYXRlIjp7Im1ldGFkYXRhIjp7ImxhYmVscyI6eyJhcHAua3ViZXJuZXRlcy5pby9jb21wb25lbnQiOiJweGMiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6InNtYXJ0LXVwZGF0ZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9LCJhbm5vdGF0aW9ucyI6eyJrdWJlY3RsLmt1YmVybmV0ZXMuaW8vZGVmYXVsdC1jb250YWluZXIiOiJweGMiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiYTA3MGIxMWJiZjY2NDhhNjA2ZDc2Njc1MTYzNjRhNzQiLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6ImM5NmIzMGY3YzQ1OGVmZjFlZDA2NTFmYzEwMzI5ZGNjIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJ0bXAiLCJlbXB0eURpciI6e319LHsibmFtZSI6ImNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoic21hcnQtdXBkYXRlLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtc3NsLWludGVybmFsIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJzc2wiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6InNtYXJ0LXVwZGF0ZS1zc2wiLCJvcHRpb25hbCI6ZmFsc2V9fSx7Im5hbWUiOiJhdXRvLWNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoiYXV0by1zbWFydC11cGRhdGUtcHhjIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJ2YXVsdC1rZXlyaW5nLXNlY3JldCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoic21hcnQtdXBkYXRlLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXlzcWwtaW5pdC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtbXlzcWwtaW5pdCIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoiYmluIiwiZW1wdHlEaXIiOnt9fV0sImluaXRDb250YWluZXJzIjpbeyJuYW1lIjoicHhjLWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTM4LTcwYzA0ODU4IiwiY29tbWFuZCI6WyIvcHhjLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiZGF0YWRpciIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn0seyJuYW1lIjoiYmluIiwibW91bnRQYXRoIjoiL29wdC9wZXJjb25hIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifV0sImNvbnRhaW5lcnMiOlt7Im5hbWUiOiJweGMiLCJpbWFnZSI6InBlcmNvbmEvcGVyY29uYS14dHJhZGItY2x1c3Rlcjo4LjAuNDEtMzIuMSIsImNvbW1hbmQiOlsiL3Zhci9saWIvbXlzcWwvcHhjLWVudHJ5cG9pbnQuc2giXSwiYXJncyI6WyJteXNxbGQiXSwicG9ydHMiOlt7Im5hbWUiOiJteXNxbCIsImNvbnRhaW5lclBvcnQiOjMzMDZ9LHsibmFtZSI6InNzdCIsImNvbnRhaW5lclBvcnQiOjQ0NDR9LHsibmFtZSI6IndyaXRlLXNldCIsImNvbnRhaW5lclBvcnQiOjQ1Njd9LHsibmFtZSI6ImlzdCIsImNvbnRhaW5lclBvcnQiOjQ1Njh9LHsibmFtZSI6Im15c3FsLWFkbWluIiwiY29udGFpbmVyUG9ydCI6MzMwNjJ9LHsibmFtZSI6Im15c3FseCIsImNvbnRhaW5lclBvcnQiOjMzMDYwfV0sImVudkZyb20iOlt7InNlY3JldFJlZiI6eyJuYW1lIjoic21hcnQtdXBkYXRlLWVudi12YXJzLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX1dLCJlbnYiOlt7Im5hbWUiOiJQWENfU0VSVklDRSIsInZhbHVlIjoic21hcnQtdXBkYXRlLXB4Yy11bnJlYWR5In0seyJuYW1lIjoiTU9OSVRPUl9IT1NUIiwidmFsdWUiOiIlIn0seyJuYW1lIjoiTVlTUUxfUk9PVF9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6InJvb3QifX19LHsibmFtZSI6IlhUUkFCQUNLVVBfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJ4dHJhYmFja3VwIn19fSx7Im5hbWUiOiJNT05JVE9SX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5IjoibW9uaXRvciJ9fX0seyJuYW1lIjoiQ0xVU1RFUl9IQVNIIiwidmFsdWUiOiIzNDQ1Mjg4In0seyJuYW1lIjoiT1BFUkFUT1JfQURNSU5fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJvcGVyYXRvciJ9fX0seyJuYW1lIjoiTElWRU5FU1NfQ0hFQ0tfVElNRU9VVCIsInZhbHVlIjoiNSJ9LHsibmFtZSI6IlJFQURJTkVTU19DSEVDS19USU1FT1VUIiwidmFsdWUiOiIxNSJ9LHsibmFtZSI6IkRFRkFVTFRfQVVUSEVOVElDQVRJT05fUExVR0lOIiwidmFsdWUiOiJjYWNoaW5nX3NoYTJfcGFzc3dvcmQi"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzbWFydC11cGRhdGUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifX0sInRlbXBsYXRlIjp7Im1ldGFkYXRhIjp7ImxhYmVscyI6eyJhcHAua3ViZXJuZXRlcy5pby9jb21wb25lbnQiOiJweGMiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6InNtYXJ0LXVwZGF0ZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9LCJhbm5vdGF0aW9ucyI6eyJrdWJlY3RsLmt1YmVybmV0ZXMuaW8vZGVmYXVsdC1jb250YWluZXIiOiJweGMiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiYTA3MGIxMWJiZjY2NDhhNjA2ZDc2Njc1MTYzNjRhNzQiLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6ImM5NmIzMGY3YzQ1OGVmZjFlZDA2NTFmYzEwMzI5ZGNjIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJ0bXAiLCJlbXB0eURpciI6e319LHsibmFtZSI6ImNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoic21hcnQtdXBkYXRlLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtc3NsLWludGVybmFsIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJzc2wiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6InNtYXJ0LXVwZGF0ZS1zc2wiLCJvcHRpb25hbCI6ZmFsc2V9fSx7Im5hbWUiOiJhdXRvLWNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoiYXV0by1zbWFydC11cGRhdGUtcHhjIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJ2YXVsdC1rZXlyaW5nLXNlY3JldCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoic21hcnQtdXBkYXRlLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXlzcWwtaW5pdC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtbXlzcWwtaW5pdCIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoiYmluIiwiZW1wdHlEaXIiOnt9fV0sImluaXRDb250YWluZXJzIjpbeyJuYW1lIjoicHhjLWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTM4LTcwYzA0ODU4IiwiY29tbWFuZCI6WyIvcHhjLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiZGF0YWRpciIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn0seyJuYW1lIjoiYmluIiwibW91bnRQYXRoIjoiL29wdC9wZXJjb25hIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifV0sImNvbnRhaW5lcnMiOlt7Im5hbWUiOiJweGMiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjptYWluLXB4YzguMCIsImNvbW1hbmQiOlsiL3Zhci9saWIvbXlzcWwvcHhjLWVudHJ5cG9pbnQuc2giXSwiYXJncyI6WyJteXNxbGQiXSwicG9ydHMiOlt7Im5hbWUiOiJteXNxbCIsImNvbnRhaW5lclBvcnQiOjMzMDZ9LHsibmFtZSI6InNzdCIsImNvbnRhaW5lclBvcnQiOjQ0NDR9LHsibmFtZSI6IndyaXRlLXNldCIsImNvbnRhaW5lclBvcnQiOjQ1Njd9LHsibmFtZSI6ImlzdCIsImNvbnRhaW5lclBvcnQiOjQ1Njh9LHsibmFtZSI6Im15c3FsLWFkbWluIiwiY29udGFpbmVyUG9ydCI6MzMwNjJ9LHsibmFtZSI6Im15c3FseCIsImNvbnRhaW5lclBvcnQiOjMzMDYwfV0sImVudkZyb20iOlt7InNlY3JldFJlZiI6eyJuYW1lIjoic21hcnQtdXBkYXRlLWVudi12YXJzLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX1dLCJlbnYiOlt7Im5hbWUiOiJQWENfU0VSVklDRSIsInZhbHVlIjoic21hcnQtdXBkYXRlLXB4Yy11bnJlYWR5In0seyJuYW1lIjoiTU9OSVRPUl9IT1NUIiwidmFsdWUiOiIlIn0seyJuYW1lIjoiTVlTUUxfUk9PVF9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6InJvb3QifX19LHsibmFtZSI6IlhUUkFCQUNLVVBfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJ4dHJhYmFja3VwIn19fSx7Im5hbWUiOiJNT05JVE9SX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5IjoibW9uaXRvciJ9fX0seyJuYW1lIjoiQ0xVU1RFUl9IQVNIIiwidmFsdWUiOiIyNjg0MTExIn0seyJuYW1lIjoiT1BFUkFUT1JfQURNSU5fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJvcGVyYXRvciJ9fX0seyJuYW1lIjoiTElWRU5FU1NfQ0hFQ0tfVElNRU9VVCIsInZhbHVlIjoiNSJ9LHsibmFtZSI6IlJFQURJTkVTU19DSEVDS19USU1FT1VUIiwidmFsdWUiOiIxNSJ9LHsibmFtZSI6IkRFRkFVTFRfQVVUSEVOVElDQVRJT05fUExVR0lOIiwidmFsdWUiOiJjYWNoaW5nX3No"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzbWFydC11cGRhdGUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifX0sInRlbXBsYXRlIjp7Im1ldGFkYXRhIjp7ImxhYmVscyI6eyJhcHAua3ViZXJuZXRlcy5pby9jb21wb25lbnQiOiJweGMiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6InNtYXJ0LXVwZGF0ZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9LCJhbm5vdGF0aW9ucyI6eyJrdWJlY3RsLmt1YmVybmV0ZXMuaW8vZGVmYXVsdC1jb250YWluZXIiOiJweGMiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiYTA3MGIxMWJiZjY2NDhhNjA2ZDc2Njc1MTYzNjRhNzQiLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6ImM5NmIzMGY3YzQ1OGVmZjFlZDA2NTFmYzEwMzI5ZGNjIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJ0bXAiLCJlbXB0eURpciI6e319LHsibmFtZSI6ImNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoic21hcnQtdXBkYXRlLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtc3NsLWludGVybmFsIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJzc2wiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6InNtYXJ0LXVwZGF0ZS1zc2wiLCJvcHRpb25hbCI6ZmFsc2V9fSx7Im5hbWUiOiJhdXRvLWNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoiYXV0by1zbWFydC11cGRhdGUtcHhjIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJ2YXVsdC1rZXlyaW5nLXNlY3JldCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoic21hcnQtdXBkYXRlLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXlzcWwtaW5pdC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtbXlzcWwtaW5pdCIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoiYmluIiwiZW1wdHlEaXIiOnt9fV0sImluaXRDb250YWluZXJzIjpbeyJuYW1lIjoicHhjLWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTM4LTcwYzA0ODU4IiwiY29tbWFuZCI6WyIvcHhjLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiZGF0YWRpciIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn0seyJuYW1lIjoiYmluIiwibW91bnRQYXRoIjoiL29wdC9wZXJjb25hIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifV0sImNvbnRhaW5lcnMiOlt7Im5hbWUiOiJweGMiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjptYWluLXB4YzguMCIsImNvbW1hbmQiOlsiL3Zhci9saWIvbXlzcWwvcHhjLWVudHJ5cG9pbnQuc2giXSwiYXJncyI6WyJteXNxbGQiXSwicG9ydHMiOlt7Im5hbWUiOiJteXNxbCIsImNvbnRhaW5lclBvcnQiOjMzMDZ9LHsibmFtZSI6InNzdCIsImNvbnRhaW5lclBvcnQiOjQ0NDR9LHsibmFtZSI6IndyaXRlLXNldCIsImNvbnRhaW5lclBvcnQiOjQ1Njd9LHsibmFtZSI6ImlzdCIsImNvbnRhaW5lclBvcnQiOjQ1Njh9LHsibmFtZSI6Im15c3FsLWFkbWluIiwiY29udGFpbmVyUG9ydCI6MzMwNjJ9LHsibmFtZSI6Im15c3FseCIsImNvbnRhaW5lclBvcnQiOjMzMDYwfV0sImVudkZyb20iOlt7InNlY3JldFJlZiI6eyJuYW1lIjoic21hcnQtdXBkYXRlLWVudi12YXJzLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX1dLCJlbnYiOlt7Im5hbWUiOiJQWENfU0VSVklDRSIsInZhbHVlIjoic21hcnQtdXBkYXRlLXB4Yy11bnJlYWR5In0seyJuYW1lIjoiTU9OSVRPUl9IT1NUIiwidmFsdWUiOiIlIn0seyJuYW1lIjoiTVlTUUxfUk9PVF9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6InJvb3QifX19LHsibmFtZSI6IlhUUkFCQUNLVVBfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJ4dHJhYmFja3VwIn19fSx7Im5hbWUiOiJNT05JVE9SX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5IjoibW9uaXRvciJ9fX0seyJuYW1lIjoiQ0xVU1RFUl9IQVNIIiwidmFsdWUiOiIzMzI3NDExIn0seyJuYW1lIjoiT1BFUkFUT1JfQURNSU5fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJvcGVyYXRvciJ9fX0seyJuYW1lIjoiTElWRU5FU1NfQ0hFQ0tfVElNRU9VVCIsInZhbHVlIjoiNSJ9LHsibmFtZSI6IlJFQURJTkVTU19DSEVDS19USU1FT1VUIiwidmFsdWUiOiIxNSJ9LHsibmFtZSI6IkRFRkFVTFRfQVVUSEVOVElDQVRJT05fUExVR0lOIiwidmFsdWUiOiJjYWNoaW5nX3No"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzbWFydC11cGRhdGUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifX0sInRlbXBsYXRlIjp7Im1ldGFkYXRhIjp7ImxhYmVscyI6eyJhcHAua3ViZXJuZXRlcy5pby9jb21wb25lbnQiOiJweGMiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6InNtYXJ0LXVwZGF0ZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9LCJhbm5vdGF0aW9ucyI6eyJrdWJlY3RsLmt1YmVybmV0ZXMuaW8vZGVmYXVsdC1jb250YWluZXIiOiJweGMiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiYTA3MGIxMWJiZjY2NDhhNjA2ZDc2Njc1MTYzNjRhNzQiLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6ImM5NmIzMGY3YzQ1OGVmZjFlZDA2NTFmYzEwMzI5ZGNjIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJ0bXAiLCJlbXB0eURpciI6e319LHsibmFtZSI6ImNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoic21hcnQtdXBkYXRlLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtc3NsLWludGVybmFsIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJzc2wiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6InNtYXJ0LXVwZGF0ZS1zc2wiLCJvcHRpb25hbCI6ZmFsc2V9fSx7Im5hbWUiOiJhdXRvLWNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoiYXV0by1zbWFydC11cGRhdGUtcHhjIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJ2YXVsdC1rZXlyaW5nLXNlY3JldCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoic21hcnQtdXBkYXRlLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXlzcWwtaW5pdC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtbXlzcWwtaW5pdCIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoiYmluIiwiZW1wdHlEaXIiOnt9fV0sImluaXRDb250YWluZXJzIjpbeyJuYW1lIjoicHhjLWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTM4LTcwYzA0ODU4IiwiY29tbWFuZCI6WyIvcHhjLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiZGF0YWRpciIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn0seyJuYW1lIjoiYmluIiwibW91bnRQYXRoIjoiL29wdC9wZXJjb25hIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifV0sImNvbnRhaW5lcnMiOlt7Im5hbWUiOiJweGMiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjptYWluLXB4YzguMCIsImNvbW1hbmQiOlsiL3Zhci9saWIvbXlzcWwvcHhjLWVudHJ5cG9pbnQuc2giXSwiYXJncyI6WyJteXNxbGQiXSwicG9ydHMiOlt7Im5hbWUiOiJteXNxbCIsImNvbnRhaW5lclBvcnQiOjMzMDZ9LHsibmFtZSI6InNzdCIsImNvbnRhaW5lclBvcnQiOjQ0NDR9LHsibmFtZSI6IndyaXRlLXNldCIsImNvbnRhaW5lclBvcnQiOjQ1Njd9LHsibmFtZSI6ImlzdCIsImNvbnRhaW5lclBvcnQiOjQ1Njh9LHsibmFtZSI6Im15c3FsLWFkbWluIiwiY29udGFpbmVyUG9ydCI6MzMwNjJ9LHsibmFtZSI6Im15c3FseCIsImNvbnRhaW5lclBvcnQiOjMzMDYwfV0sImVudkZyb20iOlt7InNlY3JldFJlZiI6eyJuYW1lIjoic21hcnQtdXBkYXRlLWVudi12YXJzLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX1dLCJlbnYiOlt7Im5hbWUiOiJQWENfU0VSVklDRSIsInZhbHVlIjoic21hcnQtdXBkYXRlLXB4Yy11bnJlYWR5In0seyJuYW1lIjoiTU9OSVRPUl9IT1NUIiwidmFsdWUiOiIlIn0seyJuYW1lIjoiTVlTUUxfUk9PVF9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6InJvb3QifX19LHsibmFtZSI6IlhUUkFCQUNLVVBfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJ4dHJhYmFja3VwIn19fSx7Im5hbWUiOiJNT05JVE9SX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5IjoibW9uaXRvciJ9fX0seyJuYW1lIjoiQ0xVU1RFUl9IQVNIIiwidmFsdWUiOiIzNDQ1Mjg4In0seyJuYW1lIjoiT1BFUkFUT1JfQURNSU5fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJvcGVyYXRvciJ9fX0seyJuYW1lIjoiTElWRU5FU1NfQ0hFQ0tfVElNRU9VVCIsInZhbHVlIjoiNSJ9LHsibmFtZSI6IlJFQURJTkVTU19DSEVDS19USU1FT1VUIiwidmFsdWUiOiIxNSJ9LHsibmFtZSI6IkRFRkFVTFRfQVVUSEVOVElDQVRJT05fUExVR0lOIiwidmFsdWUiOiJjYWNoaW5nX3No"..., "/percona-xtradb-cluster", + PeriodSeconds: 0, - PeriodSeconds: 10, + PersistentVolumeClaimRetentionPolicy: nil, - PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}", + Phase: "", - Phase: "Pending", + PodManagementPolicy: "", - PodManagementPolicy: "OrderedReady", Ports: []v1.ContainerPort{ PreemptionPolicy: nil, ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}}, + Protocol: "", - Protocol: "TCP", Quobyte: nil, ReadinessProbe: &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...}, + ReadyReplicas: 0, - ReadyReplicas: 3, + Replicas: 0, Replicas: &3, - Replicas: 3, + ResourceVersion: "", - ResourceVersion: "1782966152734639020", - ResourceVersion: "1782966892187279020", - ResourceVersion: "1782967568263759020", + RestartPolicy: "", - RestartPolicy: "Always", - RevisionHistoryLimit: &10, + RevisionHistoryLimit: nil, + SchedulerName: "", - SchedulerName: "default-scheduler", SecretName: "internal-smart-update", SecretName: "smart-update-mysql-init", SecretName: "smart-update-ssl", SecretName: "smart-update-ssl-internal", SecretName: "smart-update-vault", Secret: &v1.SecretVolumeSource{ SecurityContext: nil, Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "smart-update", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, SelfLink: "", ServiceAccountName: "default", ServiceName: "smart-update-pxc", SetHostnameAsFQDN: nil, sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 Spec: v1.PersistentVolumeClaimSpec{ Spec: v1.PodSpec{ Spec: v1.StatefulSetSpec{ StartupProbe: nil, Status: v1.PersistentVolumeClaimStatus{ Status: v1.StatefulSetStatus{ StorageClassName: nil, Subdomain: "", - Subresource: "status", SuccessThreshold: 1, Template: v1.PodTemplateSpec{ TerminationGracePeriodSeconds: &600, TerminationGracePeriodSeconds: nil, + TerminationMessagePath: "", - TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "", - TerminationMessagePolicy: "File", TimeoutSeconds: 5, - Time: s"2026-07-02 04:19:36 +0000 UTC", - Time: s"2026-07-02 04:22:32 +0000 UTC", - Time: s"2026-07-02 04:30:56 +0000 UTC", - Time: s"2026-07-02 04:34:52 +0000 UTC", - Time: s"2026-07-02 04:42:14 +0000 UTC", - Time: s"2026-07-02 04:46:08 +0000 UTC", Tolerations: nil, - TopologySpreadConstraints: nil, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, TypeMeta: {}, TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"}, + UID: "", - UID: "6f4cc0fa-fdd3-47f7-a89b-9569a53becfa", - UID: "949d863a-0362-4fb1-a001-1473c9cd99b9", - UID: "a0842826-32ec-46b1-8b6d-a0854d129cfd", + UpdatedReplicas: 0, - UpdatedReplicas: 3, + UpdateRevision: "", - UpdateRevision: "smart-update-pxc-544f97b756", - UpdateRevision: "smart-update-pxc-79c7775b5b", - UpdateRevision: "smart-update-pxc-867d655857", UpdateStrategy: {Type: "OnDelete"}, &v1.StatefulSet{ VolumeAttributesClassName: nil, VolumeClaimTemplates: []v1.PersistentVolumeClaim{ VolumeDevices: nil, - VolumeMode: &"Filesystem", + VolumeMode: nil, VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...}, VolumeName: "", VolumeSource: v1.VolumeSource{ Volumes: []v1.Volume{ VsphereVolume: nil, WorkingDir: "", + local backup_names ++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + backup_names= + [[ -n '' ]] + local pxc_names ++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + pxc_names= + [[ -n '' ]] + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.84Vic9TCyg ++ mktemp + local LAST_ERR=/tmp/tmp.yrtLwDySTD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.84Vic9TCyg validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.yrtLwDySTD + rm /tmp/tmp.84Vic9TCyg /tmp/tmp.yrtLwDySTD + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml namespace "cert-manager" deleted customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted serviceaccount "cert-manager-cainjector" deleted from cert-manager namespace serviceaccount "cert-manager" deleted from cert-manager namespace serviceaccount "cert-manager-webhook" deleted from cert-manager namespace clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace role.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace service "cert-manager-cainjector" deleted from cert-manager namespace service "cert-manager" deleted from cert-manager namespace service "cert-manager-webhook" deleted from cert-manager namespace deployment.apps "cert-manager-cainjector" deleted from cert-manager namespace deployment.apps "cert-manager" deleted from cert-manager namespace deployment.apps "cert-manager-webhook" deleted from cert-manager namespace mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ date +%s + kubectl_bin delete --grace-period=0 --force=true namespace smart-update1-19768 + local deadline=1782968240 ++ mktemp + local LAST_OUT=/tmp/tmp.OLyB9tDJLV ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.rZq1kK9xAp + local LAST_ERR=/tmp/tmp.5jDSzGUdJa + local exit_status=0 ++ mktemp + local LAST_ERR=/tmp/tmp.KSbkMWKL0u + local exit_status=0 ++ kubectl_bin get namespace smart-update1-19768 --ignore-not-found -o name ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace smart-update1-19768 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + [[ -z namespace/smart-update1-19768 ]] ++ date +%s + [[ 1782968123 -ge 1782968240 ]] + sleep 5 ++ kubectl_bin get namespace smart-update1-19768 --ignore-not-found -o name + [[ -z namespace/smart-update1-19768 ]] ++ date +%s + [[ 1782968130 -ge 1782968240 ]] + sleep 5 ++ kubectl_bin get namespace smart-update1-19768 --ignore-not-found -o name + [[ -z namespace/smart-update1-19768 ]] ++ date +%s + [[ 1782968138 -ge 1782968240 ]] + sleep 5 ++ kubectl_bin get namespace smart-update1-19768 --ignore-not-found -o name + [[ -z namespace/smart-update1-19768 ]] ++ date +%s + [[ 1782968145 -ge 1782968240 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rZq1kK9xAp namespace "smart-update1-19768" force deleted + cat /tmp/tmp.KSbkMWKL0u Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.rZq1kK9xAp /tmp/tmp.KSbkMWKL0u + return 0 ++ kubectl_bin get namespace smart-update1-19768 --ignore-not-found -o name + [[ -z '' ]] ++ date +%s + deadline=1782968272 ++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' ++ awk '$2 != ""' + [[ -z '' ]] + rm -rf /tmp/tmp.Cj53gF286S + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------