Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/logs/smart-update1-8-4.log Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 + API=pxc.percona.com/v9-9-9 + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + CLUSTER=smart-update + CLUSTER_SIZE=3 + PROXY_SIZE=2 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.4 ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' + PXC_VER=8.4 + TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + VS_URL=http://version-service + VS_PORT=11000 + VS_ENDPOINT=http://version-service:11000 + main + create_infra smart-update1-11246 + local ns=smart-update1-11246 + '[' -n pxc-operator ']' + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide + kubectl patch pxc -n smart-update1-8511 smart-update --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/smart-update patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.gpK4QyImOf ++ mktemp + local LAST_ERR=/tmp/tmp.0ubjUoYURl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gpK4QyImOf perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update1-8511 namespace + cat /tmp/tmp.0ubjUoYURl + rm /tmp/tmp.gpK4QyImOf /tmp/tmp.0ubjUoYURl + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.6mQ9SoQUxB ++ mktemp + local LAST_ERR=/tmp/tmp.kjBwoZwzUN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6mQ9SoQUxB No resources found + cat /tmp/tmp.kjBwoZwzUN + rm /tmp/tmp.6mQ9SoQUxB /tmp/tmp.kjBwoZwzUN + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.1KLkqLzHHA ++ mktemp + local LAST_ERR=/tmp/tmp.b6MSDyjDfj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1KLkqLzHHA No resources found + cat /tmp/tmp.b6MSDyjDfj + rm /tmp/tmp.1KLkqLzHHA /tmp/tmp.b6MSDyjDfj + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.e5iKsjUiSs + awk '{print$1}' + kubectl_bin get ns + xargs kubectl delete ns ++ mktemp + local LAST_ERR=/tmp/tmp.i1y9gkHZpP + local exit_status=0 ++ mktemp + local LAST_OUT=/tmp/tmp.Y121F9048q ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator ++ mktemp + local LAST_ERR=/tmp/tmp.MEJJWWwYlI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Y121F9048q + cat /tmp/tmp.MEJJWWwYlI + rm /tmp/tmp.Y121F9048q /tmp/tmp.MEJJWWwYlI + return 0 namespace "cert-manager" deleted namespace "smart-update1-8511" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.e5iKsjUiSs namespace "pxc-operator" deleted + cat /tmp/tmp.i1y9gkHZpP + rm /tmp/tmp.e5iKsjUiSs /tmp/tmp.i1y9gkHZpP + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.XfIPCW8MdO ++ mktemp + local LAST_ERR=/tmp/tmp.fucWhzPzMd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XfIPCW8MdO namespace/pxc-operator created + cat /tmp/tmp.fucWhzPzMd + rm /tmp/tmp.XfIPCW8MdO /tmp/tmp.fucWhzPzMd + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.oR4uH8JZHQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.zGw8OVp2dr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oR4uH8JZHQ ++ cat /tmp/tmp.zGw8OVp2dr ++ rm /tmp/tmp.oR4uH8JZHQ /tmp/tmp.zGw8OVp2dr ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster6 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.m8pG1bqh31 ++ mktemp + local LAST_ERR=/tmp/tmp.FyW0nUpZPX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster6 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.m8pG1bqh31 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster6" modified. + cat /tmp/tmp.FyW0nUpZPX + rm /tmp/tmp.m8pG1bqh31 /tmp/tmp.FyW0nUpZPX + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.bHjj8JjjDQ ++ mktemp + local LAST_ERR=/tmp/tmp.pM5ekhtBHr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bHjj8JjjDQ customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.pM5ekhtBHr + rm /tmp/tmp.bHjj8JjjDQ /tmp/tmp.pM5ekhtBHr + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: pxc-operator^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/cw-rbac.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.7NhQvLm1hM ++ mktemp + local LAST_ERR=/tmp/tmp.A5quSxU9bR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7NhQvLm1hM clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.A5quSxU9bR + rm /tmp/tmp.7NhQvLm1hM /tmp/tmp.A5quSxU9bR + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/cw-operator.yaml + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.EHrrv02cDD ++ mktemp + local LAST_ERR=/tmp/tmp.lGIPH58WCM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EHrrv02cDD deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.lGIPH58WCM + rm /tmp/tmp.EHrrv02cDD /tmp/tmp.lGIPH58WCM + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.c7AS2fDbcM ++ mktemp + local LAST_ERR=/tmp/tmp.tlykKa8clg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.c7AS2fDbcM pod/percona-xtradb-cluster-operator-665ff7485-wpndd condition met + cat /tmp/tmp.tlykKa8clg + rm /tmp/tmp.c7AS2fDbcM /tmp/tmp.tlykKa8clg + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.YejlqnHQvW +++ mktemp ++ local LAST_ERR=/tmp/tmp.gyS2cfPsW4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YejlqnHQvW ++ cat /tmp/tmp.gyS2cfPsW4 ++ rm /tmp/tmp.YejlqnHQvW /tmp/tmp.gyS2cfPsW4 ++ return 0 + wait_pod percona-xtradb-cluster-operator-665ff7485-wpndd 480 pxc-operator + local pod=percona-xtradb-cluster-operator-665ff7485-wpndd + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-665ff7485-wpndd ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-665ff7485-wpndd condition met waiting for pod/percona-xtradb-cluster-operator-665ff7485-wpndd to become Ready.Ok + sleep 3 + create_namespace smart-update1-11246 + local namespace=smart-update1-11246 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces smart-update1-11246' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces smart-update1-11246 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace smart-update1-11246 ++ mktemp + xargs kubectl delete ns + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + local LAST_OUT=/tmp/tmp.VenJNkrEOZ ++ mktemp + local LAST_ERR=/tmp/tmp.OlmIlbZ8r0 + local exit_status=0 ++ mktemp + local LAST_OUT=/tmp/tmp.fFwGnj8ZpA ++ seq 0 2 ++ mktemp + local LAST_ERR=/tmp/tmp.CgbDqHBQPs + local exit_status=0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-11246 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fFwGnj8ZpA + cat /tmp/tmp.CgbDqHBQPs + rm /tmp/tmp.fFwGnj8ZpA /tmp/tmp.CgbDqHBQPs + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-11246 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-11246 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.VenJNkrEOZ + cat /tmp/tmp.OlmIlbZ8r0 Error from server (NotFound): namespaces "smart-update1-11246" not found + rm /tmp/tmp.VenJNkrEOZ /tmp/tmp.OlmIlbZ8r0 + return 1 + : + wait_for_delete namespace/smart-update1-11246 + local res=namespace/smart-update1-11246 + echo -n 'waiting for namespace/smart-update1-11246 to be deleted' waiting for namespace/smart-update1-11246 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "smart-update1-11246" not found + desc 'create namespace smart-update1-11246' + set +o xtrace ----------------------------------------------------------------------------------- create namespace smart-update1-11246 ----------------------------------------------------------------------------------- + kubectl_bin create namespace smart-update1-11246 ++ mktemp + local LAST_OUT=/tmp/tmp.z2J46yPQXg ++ mktemp + local LAST_ERR=/tmp/tmp.swA61gl7si + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace smart-update1-11246 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.z2J46yPQXg namespace/smart-update1-11246 created + cat /tmp/tmp.swA61gl7si + rm /tmp/tmp.z2J46yPQXg /tmp/tmp.swA61gl7si + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.63qRzm8jEh +++ mktemp ++ local LAST_ERR=/tmp/tmp.wk3m1nbYJ3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.63qRzm8jEh ++ cat /tmp/tmp.wk3m1nbYJ3 ++ rm /tmp/tmp.63qRzm8jEh /tmp/tmp.wk3m1nbYJ3 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster6 --namespace=smart-update1-11246 ++ mktemp + local LAST_OUT=/tmp/tmp.eJj1rztbNJ ++ mktemp + local LAST_ERR=/tmp/tmp.AVTBUdbI4z + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster6 --namespace=smart-update1-11246 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eJj1rztbNJ Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster6" modified. + cat /tmp/tmp.AVTBUdbI4z + rm /tmp/tmp.eJj1rztbNJ /tmp/tmp.AVTBUdbI4z + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.rW3VR8pm1N ++ mktemp + local LAST_ERR=/tmp/tmp.FiEJGR4Mxc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rW3VR8pm1N secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.FiEJGR4Mxc + rm /tmp/tmp.rW3VR8pm1N /tmp/tmp.FiEJGR4Mxc + return 0 + deploy_version_service + desc 'install version service' + set +o xtrace ----------------------------------------------------------------------------------- install version service ----------------------------------------------------------------------------------- + kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json ++ mktemp + local LAST_OUT=/tmp/tmp.or6U47kn9c ++ mktemp + local LAST_ERR=/tmp/tmp.kRc1jWGAD6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.or6U47kn9c configmap/versions created + cat /tmp/tmp.kRc1jWGAD6 + rm /tmp/tmp.or6U47kn9c /tmp/tmp.kRc1jWGAD6 + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.mXrchTcfFV ++ mktemp + local LAST_ERR=/tmp/tmp.z9Cuz1SMSI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mXrchTcfFV deployment.apps/version-service created service/version-service created + cat /tmp/tmp.z9Cuz1SMSI + rm /tmp/tmp.mXrchTcfFV /tmp/tmp.z9Cuz1SMSI + return 0 + sleep 10 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.TdxfgHyVfh ++ mktemp + local LAST_ERR=/tmp/tmp.Qpj75L4oyF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TdxfgHyVfh namespace/cert-manager created + cat /tmp/tmp.Qpj75L4oyF + rm /tmp/tmp.TdxfgHyVfh /tmp/tmp.Qpj75L4oyF + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.YOwDwSMA8M ++ mktemp + local LAST_ERR=/tmp/tmp.EYUz9v2YQx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YOwDwSMA8M namespace/cert-manager labeled + cat /tmp/tmp.EYUz9v2YQx + rm /tmp/tmp.YOwDwSMA8M /tmp/tmp.EYUz9v2YQx + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.GJmN65yCeZ ++ mktemp + local LAST_ERR=/tmp/tmp.nTfKT8STVX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GJmN65yCeZ namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.nTfKT8STVX Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.GJmN65yCeZ /tmp/tmp.nTfKT8STVX + return 0 + '[' '' == 4.10 ']' + sleep 70 ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ grep :8.4 +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++ sort -V ++ tail -n3 ++ head -n1 ++++ grep -c percona-xtradb-cluster-operator ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ head -1 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EpfZwyMeqi ++++ mktemp +++ local LAST_ERR=/tmp/tmp.dHcbdJo3Zf +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.EpfZwyMeqi +++ cat /tmp/tmp.dHcbdJo3Zf +++ rm /tmp/tmp.EpfZwyMeqi /tmp/tmp.dHcbdJo3Zf +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-665ff7485-wpndd -n pxc-operator -- curl -s http://version-service.smart-update1-11246.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.brRDjEYK5x +++ mktemp ++ local LAST_ERR=/tmp/tmp.sVNKUJ8GPa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-665ff7485-wpndd -n pxc-operator -- curl -s http://version-service.smart-update1-11246.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.brRDjEYK5x ++ cat /tmp/tmp.sVNKUJ8GPa Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.brRDjEYK5x /tmp/tmp.sVNKUJ8GPa ++ return 0 + IMAGE_PXC=percona/percona-xtradb-cluster:8.4.5-5.1 + desc 'patch crd' + set +o xtrace ----------------------------------------------------------------------------------- patch crd ----------------------------------------------------------------------------------- + kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' ++ mktemp + local LAST_OUT=/tmp/tmp.jywR4SWQTt ++ mktemp + local LAST_ERR=/tmp/tmp.8wUctIWPJC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jywR4SWQTt customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched + cat /tmp/tmp.8wUctIWPJC + rm /tmp/tmp.jywR4SWQTt /tmp/tmp.8wUctIWPJC + return 0 + desc 'Updating ProxySQL PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- Updating ProxySQL PXC cluster ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/conf/smart-update.yml /tmp/tmp.Kmewg0QhXK/smart-update.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe"' /tmp/tmp.Kmewg0QhXK/smart-update.yml + spinup_pxc smart-update /tmp/tmp.Kmewg0QhXK/smart-update.yml + local cluster=smart-update + local config=/tmp/tmp.Kmewg0QhXK/smart-update.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.RqXvSqtX2P ++ mktemp + local LAST_ERR=/tmp/tmp.3JKlQKrDdz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RqXvSqtX2P secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.3JKlQKrDdz + rm /tmp/tmp.RqXvSqtX2P /tmp/tmp.3JKlQKrDdz + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local pvc_name= + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-11246~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local LAST_OUT=/tmp/tmp.DE92ZpUJqG + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_ERR=/tmp/tmp.rLKTkK1Fk0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DE92ZpUJqG deployment.apps/pxc-client created + cat /tmp/tmp.rLKTkK1Fk0 + rm /tmp/tmp.DE92ZpUJqG /tmp/tmp.rLKTkK1Fk0 + return 0 + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + apply_config /tmp/tmp.Kmewg0QhXK/smart-update.yml + local config_file=/tmp/tmp.Kmewg0QhXK/smart-update.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.Kmewg0QhXK/smart-update.yml '' + local input_file=/tmp/tmp.Kmewg0QhXK/smart-update.yml + local pvc_name= + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-11246~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + local LAST_OUT=/tmp/tmp.xDaIsDxz3z + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + local LAST_ERR=/tmp/tmp.9Ub4kUwOCM + local exit_status=0 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' ++ seq 0 2 + cat /tmp/tmp.Kmewg0QhXK/smart-update.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xDaIsDxz3z perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.9Ub4kUwOCM + rm /tmp/tmp.xDaIsDxz3z /tmp/tmp.9Ub4kUwOCM + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tugV5bhuuQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.F9oIhT4ExP +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.tugV5bhuuQ +++ cat /tmp/tmp.F9oIhT4ExP +++ rm /tmp/tmp.tugV5bhuuQ /tmp/tmp.F9oIhT4ExP +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TZJlQxlh0u ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lGYgVIstRO +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.TZJlQxlh0u +++ cat /tmp/tmp.lGYgVIstRO +++ rm /tmp/tmp.TZJlQxlh0u /tmp/tmp.lGYgVIstRO +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-proxysql ++ return + local proxy=smart-update-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-11246 ++ mktemp + local LAST_OUT=/tmp/tmp.85B9RKN3Tl ++ mktemp + local LAST_ERR=/tmp/tmp.1eYyRGrVvi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-11246 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-11246 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-11246 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.85B9RKN3Tl + cat /tmp/tmp.1eYyRGrVvi error: no matching resources found + rm /tmp/tmp.85B9RKN3Tl /tmp/tmp.1eYyRGrVvi + return 1 + true + wait_for_running smart-update-proxysql 1 + local name=smart-update-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-proxysql-0 480 + local pod=smart-update-proxysql-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-proxysql-0 ++ grep -E '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/smart-update-proxysql-0 condition met waiting for pod/smart-update-proxysql-0 to become Ready.Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.7TnZsPKVWh +++ mktemp ++ local LAST_ERR=/tmp/tmp.O4yNWG1sI9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7TnZsPKVWh ++ cat /tmp/tmp.O4yNWG1sI9 ++ rm /tmp/tmp.7TnZsPKVWh /tmp/tmp.O4yNWG1sI9 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3UJZQdDRt2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gV3K14pKOP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3UJZQdDRt2 ++ cat /tmp/tmp.gV3K14pKOP ++ rm /tmp/tmp.3UJZQdDRt2 /tmp/tmp.gV3K14pKOP ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cMkPQVlAVb +++ mktemp ++ local LAST_ERR=/tmp/tmp.sx5bO2d7XK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cMkPQVlAVb ++ cat /tmp/tmp.sx5bO2d7XK ++ rm /tmp/tmp.cMkPQVlAVb /tmp/tmp.sx5bO2d7XK ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9dmgqolHgw +++ mktemp ++ local LAST_ERR=/tmp/tmp.TRZHDABxBR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9dmgqolHgw ++ cat /tmp/tmp.TRZHDABxBR ++ rm /tmp/tmp.9dmgqolHgw /tmp/tmp.TRZHDABxBR ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WM0ZPknV7Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.cduNpMGAK4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WM0ZPknV7Z ++ cat /tmp/tmp.cduNpMGAK4 ++ rm /tmp/tmp.WM0ZPknV7Z /tmp/tmp.cduNpMGAK4 ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HKxbEd7j1u +++ mktemp ++ local LAST_ERR=/tmp/tmp.3LweMMyH09 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HKxbEd7j1u ++ cat /tmp/tmp.3LweMMyH09 ++ rm /tmp/tmp.HKxbEd7j1u /tmp/tmp.3LweMMyH09 ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + kubectl exec smart-update-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my + return 1 +++ get_proxy smart-update +++ local target_cluster=smart-update ++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.f6WZG7DB4V +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.GY9zyZhPk0 ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.f6WZG7DB4V ++++ cat /tmp/tmp.GY9zyZhPk0 ++++ rm /tmp/tmp.f6WZG7DB4V /tmp/tmp.GY9zyZhPk0 ++++ return 0 +++ [[ '' == \t\r\u\e ]] ++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.RrP4l5XOWx +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.wgp8HSqnmW ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.RrP4l5XOWx ++++ cat /tmp/tmp.wgp8HSqnmW ++++ rm /tmp/tmp.RrP4l5XOWx /tmp/tmp.wgp8HSqnmW ++++ return 0 +++ [[ true == \t\r\u\e ]] +++ echo smart-update-proxysql +++ return ++ get_proxy_primary '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=smart-update-proxysql-0 +++ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 proxysql +++ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' +++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' +++ local pod=smart-update-proxysql-0 +++ local container_name=proxysql +++ set +o xtrace ++ local ip=smart-update-pxc-2.smart-update-pxc.smart-update1-11246.svc.cluster.local +++ echo smart-update-pxc-2.smart-update-pxc.smart-update1-11246.svc.cluster.local +++ wc -l ++ '[' 1 '!=' 1 ']' ++ cut -d. -f1 ++ echo smart-update-pxc-2.smart-update-pxc.smart-update1-11246.svc.cluster.local + initial_primary=smart-update-pxc-2 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.4"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.VTOcQ6GWYp ++ mktemp + local LAST_ERR=/tmp/tmp.n82MbysIUM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.4"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VTOcQ6GWYp perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.n82MbysIUM + rm /tmp/tmp.VTOcQ6GWYp /tmp/tmp.n82MbysIUM + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-2 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + local cluster=smart-update + local initial_primary=smart-update-pxc-2 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + set +x Waiting for the last pod to update....................................... smart-update-pxc-2 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=300 + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sTV2eRPeuw +++ mktemp ++ local LAST_ERR=/tmp/tmp.QzDwPRLl4J ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sTV2eRPeuw ++ cat /tmp/tmp.QzDwPRLl4J ++ rm /tmp/tmp.sTV2eRPeuw /tmp/tmp.QzDwPRLl4J ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 0 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fiwe2PayMk +++ mktemp ++ local LAST_ERR=/tmp/tmp.MijDmAiXAy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fiwe2PayMk ++ cat /tmp/tmp.MijDmAiXAy ++ rm /tmp/tmp.fiwe2PayMk /tmp/tmp.MijDmAiXAy ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 1 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MC3fk9syrS +++ mktemp ++ local LAST_ERR=/tmp/tmp.hDH5l5XkOH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MC3fk9syrS ++ cat /tmp/tmp.hDH5l5XkOH ++ rm /tmp/tmp.MC3fk9syrS /tmp/tmp.hDH5l5XkOH ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 2 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wbAzTfb5bU +++ mktemp ++ local LAST_ERR=/tmp/tmp.oGAhmt98gn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wbAzTfb5bU ++ cat /tmp/tmp.oGAhmt98gn ++ rm /tmp/tmp.wbAzTfb5bU /tmp/tmp.oGAhmt98gn ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 3 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sHQh5KO5g3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.pcp9VhLF37 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sHQh5KO5g3 ++ cat /tmp/tmp.pcp9VhLF37 ++ rm /tmp/tmp.sHQh5KO5g3 /tmp/tmp.pcp9VhLF37 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 4 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aKLxnhdUHX +++ mktemp ++ local LAST_ERR=/tmp/tmp.rlsd2M9fhW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aKLxnhdUHX ++ cat /tmp/tmp.rlsd2M9fhW ++ rm /tmp/tmp.aKLxnhdUHX /tmp/tmp.rlsd2M9fhW ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 5 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8TityvXV3V +++ mktemp ++ local LAST_ERR=/tmp/tmp.08Suu9wYZ2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8TityvXV3V ++ cat /tmp/tmp.08Suu9wYZ2 ++ rm /tmp/tmp.8TityvXV3V /tmp/tmp.08Suu9wYZ2 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 6 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.k6pL27SZEk +++ mktemp ++ local LAST_ERR=/tmp/tmp.IUAsVQnFev ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.k6pL27SZEk ++ cat /tmp/tmp.IUAsVQnFev ++ rm /tmp/tmp.k6pL27SZEk /tmp/tmp.IUAsVQnFev ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 7 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XS3PTWcXtq +++ mktemp ++ local LAST_ERR=/tmp/tmp.hxEwTVRVTi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XS3PTWcXtq ++ cat /tmp/tmp.hxEwTVRVTi ++ rm /tmp/tmp.XS3PTWcXtq /tmp/tmp.hxEwTVRVTi ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 8 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9nE5zmWd0N +++ mktemp ++ local LAST_ERR=/tmp/tmp.BpH9KkUPZG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9nE5zmWd0N ++ cat /tmp/tmp.BpH9KkUPZG ++ rm /tmp/tmp.9nE5zmWd0N /tmp/tmp.BpH9KkUPZG ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 9 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YFLVsQ0CqI +++ mktemp ++ local LAST_ERR=/tmp/tmp.ifPKyPVXOQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YFLVsQ0CqI ++ cat /tmp/tmp.ifPKyPVXOQ ++ rm /tmp/tmp.YFLVsQ0CqI /tmp/tmp.ifPKyPVXOQ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 10 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JsXcsskaYh +++ mktemp ++ local LAST_ERR=/tmp/tmp.HuC22avBN2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JsXcsskaYh ++ cat /tmp/tmp.HuC22avBN2 ++ rm /tmp/tmp.JsXcsskaYh /tmp/tmp.HuC22avBN2 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 11 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7B2PFjFyCw +++ mktemp ++ local LAST_ERR=/tmp/tmp.J9wFccBJKb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7B2PFjFyCw ++ cat /tmp/tmp.J9wFccBJKb ++ rm /tmp/tmp.7B2PFjFyCw /tmp/tmp.J9wFccBJKb ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 12 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NtMbxIuAD6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.CTImYN8ljD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NtMbxIuAD6 ++ cat /tmp/tmp.CTImYN8ljD ++ rm /tmp/tmp.NtMbxIuAD6 /tmp/tmp.CTImYN8ljD ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 13 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dr8LdfD2Kf +++ mktemp ++ local LAST_ERR=/tmp/tmp.E2UId3C5qV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dr8LdfD2Kf ++ cat /tmp/tmp.E2UId3C5qV ++ rm /tmp/tmp.dr8LdfD2Kf /tmp/tmp.E2UId3C5qV ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 14 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q65hibAZt4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.TvtUOUsBrB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.q65hibAZt4 ++ cat /tmp/tmp.TvtUOUsBrB ++ rm /tmp/tmp.q65hibAZt4 /tmp/tmp.TvtUOUsBrB ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 15 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.R6dSeGFFMd +++ mktemp ++ local LAST_ERR=/tmp/tmp.SXZbNno2oS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.R6dSeGFFMd ++ cat /tmp/tmp.SXZbNno2oS ++ rm /tmp/tmp.R6dSeGFFMd /tmp/tmp.SXZbNno2oS ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 16 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9q03cRbmc0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.n1CytqAVxK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9q03cRbmc0 ++ cat /tmp/tmp.n1CytqAVxK ++ rm /tmp/tmp.9q03cRbmc0 /tmp/tmp.n1CytqAVxK ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 17 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BV0xTXkIDP +++ mktemp ++ local LAST_ERR=/tmp/tmp.Or8LpyZgD5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BV0xTXkIDP ++ cat /tmp/tmp.Or8LpyZgD5 ++ rm /tmp/tmp.BV0xTXkIDP /tmp/tmp.Or8LpyZgD5 ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GvnGgTM6IL +++ mktemp ++ local LAST_ERR=/tmp/tmp.65qZQTTIL5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GvnGgTM6IL ++ cat /tmp/tmp.65qZQTTIL5 ++ rm /tmp/tmp.GvnGgTM6IL /tmp/tmp.65qZQTTIL5 ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.JbPjFcJhWZ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.30HRYLOAU1 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.JbPjFcJhWZ +++++ cat /tmp/tmp.30HRYLOAU1 +++++ rm /tmp/tmp.JbPjFcJhWZ /tmp/tmp.30HRYLOAU1 +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.WTXhuBNkMp ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Ko73tWWNEc +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.WTXhuBNkMp +++++ cat /tmp/tmp.Ko73tWWNEc +++++ rm /tmp/tmp.WTXhuBNkMp /tmp/tmp.Ko73tWWNEc +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-proxysql ++++ return +++ local cluster_proxy=smart-update-proxysql +++ echo proxysql ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GfCOTirLZu +++ mktemp ++ local LAST_ERR=/tmp/tmp.7gl3oiOShE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GfCOTirLZu ++ cat /tmp/tmp.7gl3oiOShE ++ rm /tmp/tmp.GfCOTirLZu /tmp/tmp.7gl3oiOShE ++ return 0 + [[ 2 == \2 ]] + echo ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4MXfzRexTx +++ mktemp ++ local LAST_ERR=/tmp/tmp.uaz9oPvLwF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4MXfzRexTx ++ cat /tmp/tmp.uaz9oPvLwF ++ rm /tmp/tmp.4MXfzRexTx /tmp/tmp.uaz9oPvLwF ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6NK3aRcJJP +++ mktemp ++ local LAST_ERR=/tmp/tmp.egL05LFxAR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6NK3aRcJJP ++ cat /tmp/tmp.egL05LFxAR ++ rm /tmp/tmp.6NK3aRcJJP /tmp/tmp.egL05LFxAR ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.M6sOloe92x +++ mktemp ++ local LAST_ERR=/tmp/tmp.WpIItWKxWs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.M6sOloe92x ++ cat /tmp/tmp.WpIItWKxWs ++ rm /tmp/tmp.M6sOloe92x /tmp/tmp.WpIItWKxWs ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/conf/smart-update.yml ++ mktemp + local LAST_OUT=/tmp/tmp.0KSlJFTvxy ++ mktemp + local LAST_ERR=/tmp/tmp.SBMwNTsKX5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/conf/smart-update.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0KSlJFTvxy perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update1-11246 namespace + cat /tmp/tmp.SBMwNTsKX5 + rm /tmp/tmp.0KSlJFTvxy /tmp/tmp.SBMwNTsKX5 + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.K9vmBdvZIK ++ mktemp + local LAST_ERR=/tmp/tmp.Pv1w3xW84l + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.K9vmBdvZIK persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update1-11246 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update1-11246 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update1-11246 namespace persistentvolumeclaim "proxydata-smart-update-proxysql-0" deleted from smart-update1-11246 namespace persistentvolumeclaim "proxydata-smart-update-proxysql-1" deleted from smart-update1-11246 namespace + cat /tmp/tmp.Pv1w3xW84l + rm /tmp/tmp.K9vmBdvZIK /tmp/tmp.Pv1w3xW84l + return 0 + desc 'Updating HAProxy PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- Updating HAProxy PXC cluster ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/conf/smart-update-haproxy.yml /tmp/tmp.Kmewg0QhXK/smart-update-haproxy.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe"' /tmp/tmp.Kmewg0QhXK/smart-update-haproxy.yml + spinup_pxc smart-update /tmp/tmp.Kmewg0QhXK/smart-update-haproxy.yml + local cluster=smart-update + local config=/tmp/tmp.Kmewg0QhXK/smart-update-haproxy.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.19P3UFrQWa ++ mktemp + local LAST_ERR=/tmp/tmp.pfjOU6X2Xx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.19P3UFrQWa secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.pfjOU6X2Xx + rm /tmp/tmp.19P3UFrQWa /tmp/tmp.pfjOU6X2Xx + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local pvc_name= ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.GzXWg7Mh6O + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-11246~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml ++ mktemp + local LAST_ERR=/tmp/tmp.PhwWR8ZpKq + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GzXWg7Mh6O deployment.apps/pxc-client unchanged + cat /tmp/tmp.PhwWR8ZpKq + rm /tmp/tmp.GzXWg7Mh6O /tmp/tmp.PhwWR8ZpKq + return 0 + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + apply_config /tmp/tmp.Kmewg0QhXK/smart-update-haproxy.yml + local config_file=/tmp/tmp.Kmewg0QhXK/smart-update-haproxy.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.Kmewg0QhXK/smart-update-haproxy.yml '' + local input_file=/tmp/tmp.Kmewg0QhXK/smart-update-haproxy.yml + local pvc_name= + cat /tmp/tmp.Kmewg0QhXK/smart-update-haproxy.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-11246~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + local LAST_OUT=/tmp/tmp.SSDG77yTje ++ mktemp + local LAST_ERR=/tmp/tmp.cqu3gJvo2k + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SSDG77yTje perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.cqu3gJvo2k + rm /tmp/tmp.SSDG77yTje /tmp/tmp.cqu3gJvo2k + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qKod0urHIc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uZ9R0G0Ct7 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.qKod0urHIc +++ cat /tmp/tmp.uZ9R0G0Ct7 +++ rm /tmp/tmp.qKod0urHIc /tmp/tmp.uZ9R0G0Ct7 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-11246 ++ mktemp + local LAST_OUT=/tmp/tmp.tOrw1wrmdt ++ mktemp + local LAST_ERR=/tmp/tmp.PgNwruI449 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-11246 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-11246 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-11246 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.tOrw1wrmdt + cat /tmp/tmp.PgNwruI449 error: no matching resources found + rm /tmp/tmp.tOrw1wrmdt /tmp/tmp.PgNwruI449 + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met waiting for pod/smart-update-haproxy-0 to become Ready.Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.sjUVm5qdvr +++ mktemp ++ local LAST_ERR=/tmp/tmp.LZfM38oRRj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sjUVm5qdvr ++ cat /tmp/tmp.LZfM38oRRj ++ rm /tmp/tmp.sjUVm5qdvr /tmp/tmp.LZfM38oRRj ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.O1NdBnltaa +++ mktemp ++ local LAST_ERR=/tmp/tmp.8StjnTpJkO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.O1NdBnltaa ++ cat /tmp/tmp.8StjnTpJkO ++ rm /tmp/tmp.O1NdBnltaa /tmp/tmp.8StjnTpJkO ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Asl3b47YN8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.lySpBa4Or6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Asl3b47YN8 ++ cat /tmp/tmp.lySpBa4Or6 ++ rm /tmp/tmp.Asl3b47YN8 /tmp/tmp.lySpBa4Or6 ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1ZrghfqfzH +++ mktemp ++ local LAST_ERR=/tmp/tmp.15B10UJkeV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1ZrghfqfzH ++ cat /tmp/tmp.15B10UJkeV ++ rm /tmp/tmp.1ZrghfqfzH /tmp/tmp.15B10UJkeV ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d0vlOZS90i +++ mktemp ++ local LAST_ERR=/tmp/tmp.awxX5KJJ66 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.d0vlOZS90i ++ cat /tmp/tmp.awxX5KJJ66 ++ rm /tmp/tmp.d0vlOZS90i /tmp/tmp.awxX5KJJ66 ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yDyXxgvc67 +++ mktemp ++ local LAST_ERR=/tmp/tmp.tHQogRjvgW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yDyXxgvc67 ++ cat /tmp/tmp.tHQogRjvgW ++ rm /tmp/tmp.yDyXxgvc67 /tmp/tmp.tHQogRjvgW ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + kubectl exec smart-update-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my + return 1 ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.i3rZMYhPIz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.L7ZDBbMksi +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.i3rZMYhPIz +++ cat /tmp/tmp.L7ZDBbMksi +++ rm /tmp/tmp.i3rZMYhPIz /tmp/tmp.L7ZDBbMksi +++ return 0 ++ client_pod=pxc-client-58956d9c99-8ltqs ++ wait_pod pxc-client-58956d9c99-8ltqs ++ local pod=pxc-client-58956d9c99-8ltqs ++ local max_retry=480 ++ local ns= +++ echo pxc-client-58956d9c99-8ltqs +++ grep -E '^(pxc|proxysql)$' +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ local container= ++ set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.4"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.OqSpQFZwIr ++ mktemp + local LAST_ERR=/tmp/tmp.7w8zpyEYZt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.4"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OqSpQFZwIr perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.7w8zpyEYZt + rm /tmp/tmp.OqSpQFZwIr /tmp/tmp.7w8zpyEYZt + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + set +x Waiting for the last pod to update...................... smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=300 + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XMUOC9u4i2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.CpOXm2UHDR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XMUOC9u4i2 ++ cat /tmp/tmp.CpOXm2UHDR ++ rm /tmp/tmp.XMUOC9u4i2 /tmp/tmp.CpOXm2UHDR ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 0 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5CIjotrmEh +++ mktemp ++ local LAST_ERR=/tmp/tmp.IYkvPEDmqv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5CIjotrmEh ++ cat /tmp/tmp.IYkvPEDmqv ++ rm /tmp/tmp.5CIjotrmEh /tmp/tmp.IYkvPEDmqv ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 1 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MrmL5AQO0B +++ mktemp ++ local LAST_ERR=/tmp/tmp.bCvqwf99nU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MrmL5AQO0B ++ cat /tmp/tmp.bCvqwf99nU ++ rm /tmp/tmp.MrmL5AQO0B /tmp/tmp.bCvqwf99nU ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 2 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U2LbT4h67W +++ mktemp ++ local LAST_ERR=/tmp/tmp.k6OEcUZS8V ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.U2LbT4h67W ++ cat /tmp/tmp.k6OEcUZS8V ++ rm /tmp/tmp.U2LbT4h67W /tmp/tmp.k6OEcUZS8V ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 3 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EvEU0P4x6p +++ mktemp ++ local LAST_ERR=/tmp/tmp.qjOgVncdWA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EvEU0P4x6p ++ cat /tmp/tmp.qjOgVncdWA ++ rm /tmp/tmp.EvEU0P4x6p /tmp/tmp.qjOgVncdWA ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 4 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.grwQvpIP6u +++ mktemp ++ local LAST_ERR=/tmp/tmp.rGtT87klz3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.grwQvpIP6u ++ cat /tmp/tmp.rGtT87klz3 ++ rm /tmp/tmp.grwQvpIP6u /tmp/tmp.rGtT87klz3 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 5 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XNB8uQJmSS +++ mktemp ++ local LAST_ERR=/tmp/tmp.qg71RGOJWM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XNB8uQJmSS ++ cat /tmp/tmp.qg71RGOJWM ++ rm /tmp/tmp.XNB8uQJmSS /tmp/tmp.qg71RGOJWM ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 6 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OG1VFGxIRk +++ mktemp ++ local LAST_ERR=/tmp/tmp.zheFcHcb5a ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OG1VFGxIRk ++ cat /tmp/tmp.zheFcHcb5a ++ rm /tmp/tmp.OG1VFGxIRk /tmp/tmp.zheFcHcb5a ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 7 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ABkbYZjoQM +++ mktemp ++ local LAST_ERR=/tmp/tmp.vpG684iOwf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ABkbYZjoQM ++ cat /tmp/tmp.vpG684iOwf ++ rm /tmp/tmp.ABkbYZjoQM /tmp/tmp.vpG684iOwf ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 8 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4DsLZgJaUs +++ mktemp ++ local LAST_ERR=/tmp/tmp.oStVxF5jxw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4DsLZgJaUs ++ cat /tmp/tmp.oStVxF5jxw ++ rm /tmp/tmp.4DsLZgJaUs /tmp/tmp.oStVxF5jxw ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 9 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5fjTli8EG1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.3mMYAV8MxK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5fjTli8EG1 ++ cat /tmp/tmp.3mMYAV8MxK ++ rm /tmp/tmp.5fjTli8EG1 /tmp/tmp.3mMYAV8MxK ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 10 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5yFvPKz0Yb +++ mktemp ++ local LAST_ERR=/tmp/tmp.lrwmc3HImP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5yFvPKz0Yb ++ cat /tmp/tmp.lrwmc3HImP ++ rm /tmp/tmp.5yFvPKz0Yb /tmp/tmp.lrwmc3HImP ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 11 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bZyL9Qju1Y +++ mktemp ++ local LAST_ERR=/tmp/tmp.HQ9RWdG8JP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bZyL9Qju1Y ++ cat /tmp/tmp.HQ9RWdG8JP ++ rm /tmp/tmp.bZyL9Qju1Y /tmp/tmp.HQ9RWdG8JP ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 12 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EYwfQu37Nx +++ mktemp ++ local LAST_ERR=/tmp/tmp.YfdGQbnHrY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EYwfQu37Nx ++ cat /tmp/tmp.YfdGQbnHrY ++ rm /tmp/tmp.EYwfQu37Nx /tmp/tmp.YfdGQbnHrY ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 13 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GINDabfWhW +++ mktemp ++ local LAST_ERR=/tmp/tmp.FRAClIPtgW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GINDabfWhW ++ cat /tmp/tmp.FRAClIPtgW ++ rm /tmp/tmp.GINDabfWhW /tmp/tmp.FRAClIPtgW ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 14 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tQN13Snc18 +++ mktemp ++ local LAST_ERR=/tmp/tmp.F2hs9Pspga ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tQN13Snc18 ++ cat /tmp/tmp.F2hs9Pspga ++ rm /tmp/tmp.tQN13Snc18 /tmp/tmp.F2hs9Pspga ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 15 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w3AmkZ7jSU +++ mktemp ++ local LAST_ERR=/tmp/tmp.a2AhAy33ks ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.w3AmkZ7jSU ++ cat /tmp/tmp.a2AhAy33ks ++ rm /tmp/tmp.w3AmkZ7jSU /tmp/tmp.a2AhAy33ks ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ob1lTZp2zX +++ mktemp ++ local LAST_ERR=/tmp/tmp.PSJF528cQW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ob1lTZp2zX ++ cat /tmp/tmp.PSJF528cQW ++ rm /tmp/tmp.Ob1lTZp2zX /tmp/tmp.PSJF528cQW ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.0nedAfG0qs ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.QAZfq0dB4R +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.0nedAfG0qs +++++ cat /tmp/tmp.QAZfq0dB4R +++++ rm /tmp/tmp.0nedAfG0qs /tmp/tmp.QAZfq0dB4R +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bUOKYWHV5n +++ mktemp ++ local LAST_ERR=/tmp/tmp.L46qglIfjB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bUOKYWHV5n ++ cat /tmp/tmp.L46qglIfjB ++ rm /tmp/tmp.bUOKYWHV5n /tmp/tmp.L46qglIfjB ++ return 0 + [[ 2 == \2 ]] + echo ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Qp3rUIfZQU +++ mktemp ++ local LAST_ERR=/tmp/tmp.GsL2hbBJUW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Qp3rUIfZQU ++ cat /tmp/tmp.GsL2hbBJUW ++ rm /tmp/tmp.Qp3rUIfZQU /tmp/tmp.GsL2hbBJUW ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8jPjCzxctb +++ mktemp ++ local LAST_ERR=/tmp/tmp.Rwivk7u0g8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8jPjCzxctb ++ cat /tmp/tmp.Rwivk7u0g8 ++ rm /tmp/tmp.8jPjCzxctb /tmp/tmp.Rwivk7u0g8 ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-8ltqs ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nIzldA7s0L +++ mktemp ++ local LAST_ERR=/tmp/tmp.Tc9SLD9sWL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nIzldA7s0L ++ cat /tmp/tmp.Tc9SLD9sWL ++ rm /tmp/tmp.nIzldA7s0L /tmp/tmp.Tc9SLD9sWL ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + kubectl_bin delete -f /tmp/tmp.Kmewg0QhXK/smart-update-haproxy.yml ++ mktemp + local LAST_OUT=/tmp/tmp.7OdTRChuyq ++ mktemp + local LAST_ERR=/tmp/tmp.RRQXDYsESd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.Kmewg0QhXK/smart-update-haproxy.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7OdTRChuyq perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update1-11246 namespace + cat /tmp/tmp.RRQXDYsESd + rm /tmp/tmp.7OdTRChuyq /tmp/tmp.RRQXDYsESd + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.6A4CCsfpq8 ++ mktemp + local LAST_ERR=/tmp/tmp.5RHB3D2wYy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6A4CCsfpq8 persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update1-11246 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update1-11246 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update1-11246 namespace + cat /tmp/tmp.5RHB3D2wYy + rm /tmp/tmp.6A4CCsfpq8 /tmp/tmp.5RHB3D2wYy + return 0 + desc 'Updating PXC cluster with version service available but disabled' + set +o xtrace ----------------------------------------------------------------------------------- Updating PXC cluster with version service available but disabled ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/conf/smart-update-version-service-reachable.yml /tmp/tmp.Kmewg0QhXK/smart-update-version-service-reachable.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe"' /tmp/tmp.Kmewg0QhXK/smart-update-version-service-reachable.yml + spinup_pxc smart-update /tmp/tmp.Kmewg0QhXK/smart-update-version-service-reachable.yml + local cluster=smart-update + local config=/tmp/tmp.Kmewg0QhXK/smart-update-version-service-reachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.lIRYXvRxAr ++ mktemp + local LAST_ERR=/tmp/tmp.rnTafHdzYu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lIRYXvRxAr secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.rnTafHdzYu + rm /tmp/tmp.lIRYXvRxAr /tmp/tmp.rnTafHdzYu + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-11246~ + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.iIDR7glPsi ++ mktemp + local LAST_ERR=/tmp/tmp.WrFAtKy2pP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iIDR7glPsi deployment.apps/pxc-client unchanged + cat /tmp/tmp.WrFAtKy2pP + rm /tmp/tmp.iIDR7glPsi /tmp/tmp.WrFAtKy2pP + return 0 + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + apply_config /tmp/tmp.Kmewg0QhXK/smart-update-version-service-reachable.yml + local config_file=/tmp/tmp.Kmewg0QhXK/smart-update-version-service-reachable.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.Kmewg0QhXK/smart-update-version-service-reachable.yml '' + local input_file=/tmp/tmp.Kmewg0QhXK/smart-update-version-service-reachable.yml + local pvc_name= ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + local LAST_OUT=/tmp/tmp.LnMJgq8DsA + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_ERR=/tmp/tmp.3c0JAGRHLO + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-11246~ + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + cat /tmp/tmp.Kmewg0QhXK/smart-update-version-service-reachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LnMJgq8DsA perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.3c0JAGRHLO + rm /tmp/tmp.LnMJgq8DsA /tmp/tmp.3c0JAGRHLO + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yFSaRAaUIP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4iyFqK5VYw +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.yFSaRAaUIP +++ cat /tmp/tmp.4iyFqK5VYw +++ rm /tmp/tmp.yFSaRAaUIP /tmp/tmp.4iyFqK5VYw +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-11246 ++ mktemp + local LAST_OUT=/tmp/tmp.OIfZN4cmKc ++ mktemp + local LAST_ERR=/tmp/tmp.G0xPQ0CpkS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-11246 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-11246 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-11246 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.OIfZN4cmKc + cat /tmp/tmp.G0xPQ0CpkS error: no matching resources found + rm /tmp/tmp.OIfZN4cmKc /tmp/tmp.G0xPQ0CpkS + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met waiting for pod/smart-update-haproxy-0 to become Ready.Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-2 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PtzqvP01rc +++ mktemp ++ local LAST_ERR=/tmp/tmp.JNBKvAEPI3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PtzqvP01rc ++ cat /tmp/tmp.JNBKvAEPI3 ++ rm /tmp/tmp.PtzqvP01rc /tmp/tmp.JNBKvAEPI3 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Qkox2Aacr4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.3ngdnBCTfE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Qkox2Aacr4 ++ cat /tmp/tmp.3ngdnBCTfE ++ rm /tmp/tmp.Qkox2Aacr4 /tmp/tmp.3ngdnBCTfE ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ETmtlwR8VF +++ mktemp ++ local LAST_ERR=/tmp/tmp.JKCuqkezvf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ETmtlwR8VF ++ cat /tmp/tmp.JKCuqkezvf ++ rm /tmp/tmp.ETmtlwR8VF /tmp/tmp.JKCuqkezvf ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cY0o4lRDMr +++ mktemp ++ local LAST_ERR=/tmp/tmp.jUdumunmF1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cY0o4lRDMr ++ cat /tmp/tmp.jUdumunmF1 ++ rm /tmp/tmp.cY0o4lRDMr /tmp/tmp.jUdumunmF1 ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8PZUWrfY76 +++ mktemp ++ local LAST_ERR=/tmp/tmp.a8ALfw3vjz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8PZUWrfY76 ++ cat /tmp/tmp.a8ALfw3vjz ++ rm /tmp/tmp.8PZUWrfY76 /tmp/tmp.a8ALfw3vjz ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Dg9foxTJ5p +++ mktemp ++ local LAST_ERR=/tmp/tmp.VZDplYBMz1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Dg9foxTJ5p ++ cat /tmp/tmp.VZDplYBMz1 ++ rm /tmp/tmp.Dg9foxTJ5p /tmp/tmp.VZDplYBMz1 ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + kubectl exec smart-update-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my + return 1 ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9Di18V6sAq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xQAp10FvQT +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.9Di18V6sAq +++ cat /tmp/tmp.xQAp10FvQT +++ rm /tmp/tmp.9Di18V6sAq /tmp/tmp.xQAp10FvQT +++ return 0 ++ client_pod=pxc-client-58956d9c99-8ltqs ++ wait_pod pxc-client-58956d9c99-8ltqs ++ local pod=pxc-client-58956d9c99-8ltqs ++ local max_retry=480 ++ local ns= +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ echo pxc-client-58956d9c99-8ltqs +++ grep -E '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.4"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.XxymYTnqMv ++ mktemp + local LAST_ERR=/tmp/tmp.fd5FirZ144 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.4"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XxymYTnqMv perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.fd5FirZ144 + rm /tmp/tmp.XxymYTnqMv /tmp/tmp.fd5FirZ144 + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + set +x Waiting for the last pod to update................ smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=300 + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZKbjFeO7ii +++ mktemp ++ local LAST_ERR=/tmp/tmp.rjyKKATMux ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZKbjFeO7ii ++ cat /tmp/tmp.rjyKKATMux ++ rm /tmp/tmp.ZKbjFeO7ii /tmp/tmp.rjyKKATMux ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 0 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ru7gsxLw7l +++ mktemp ++ local LAST_ERR=/tmp/tmp.gkrbThSTiH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ru7gsxLw7l ++ cat /tmp/tmp.gkrbThSTiH ++ rm /tmp/tmp.ru7gsxLw7l /tmp/tmp.gkrbThSTiH ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 1 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.P1nxNlGIzJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.b5MtmOsD9s ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.P1nxNlGIzJ ++ cat /tmp/tmp.b5MtmOsD9s ++ rm /tmp/tmp.P1nxNlGIzJ /tmp/tmp.b5MtmOsD9s ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 2 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oVDk0fzWMr +++ mktemp ++ local LAST_ERR=/tmp/tmp.E71NElVGHb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oVDk0fzWMr ++ cat /tmp/tmp.E71NElVGHb ++ rm /tmp/tmp.oVDk0fzWMr /tmp/tmp.E71NElVGHb ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 3 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Tz8D68eTMn +++ mktemp ++ local LAST_ERR=/tmp/tmp.dBrh8bEf3y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Tz8D68eTMn ++ cat /tmp/tmp.dBrh8bEf3y ++ rm /tmp/tmp.Tz8D68eTMn /tmp/tmp.dBrh8bEf3y ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 4 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0FbdRiicqc +++ mktemp ++ local LAST_ERR=/tmp/tmp.tqUC74Ig1N ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0FbdRiicqc ++ cat /tmp/tmp.tqUC74Ig1N ++ rm /tmp/tmp.0FbdRiicqc /tmp/tmp.tqUC74Ig1N ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 5 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8DjUwP6GFH +++ mktemp ++ local LAST_ERR=/tmp/tmp.CMRShvheCy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8DjUwP6GFH ++ cat /tmp/tmp.CMRShvheCy ++ rm /tmp/tmp.8DjUwP6GFH /tmp/tmp.CMRShvheCy ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 6 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.phwqLmQV4G +++ mktemp ++ local LAST_ERR=/tmp/tmp.mxPbWlvfVB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.phwqLmQV4G ++ cat /tmp/tmp.mxPbWlvfVB ++ rm /tmp/tmp.phwqLmQV4G /tmp/tmp.mxPbWlvfVB ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 7 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EwKTXYTFAi +++ mktemp ++ local LAST_ERR=/tmp/tmp.9Gw7ioJONR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EwKTXYTFAi ++ cat /tmp/tmp.9Gw7ioJONR ++ rm /tmp/tmp.EwKTXYTFAi /tmp/tmp.9Gw7ioJONR ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 8 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fXWmdLWF50 +++ mktemp ++ local LAST_ERR=/tmp/tmp.J5TSTL3y3i ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fXWmdLWF50 ++ cat /tmp/tmp.J5TSTL3y3i ++ rm /tmp/tmp.fXWmdLWF50 /tmp/tmp.J5TSTL3y3i ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 9 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CEHYnXismP +++ mktemp ++ local LAST_ERR=/tmp/tmp.sGcMfWqrsl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CEHYnXismP ++ cat /tmp/tmp.sGcMfWqrsl ++ rm /tmp/tmp.CEHYnXismP /tmp/tmp.sGcMfWqrsl ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 10 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EsOzG0Dzkx +++ mktemp ++ local LAST_ERR=/tmp/tmp.Zx1otCBoDv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EsOzG0Dzkx ++ cat /tmp/tmp.Zx1otCBoDv ++ rm /tmp/tmp.EsOzG0Dzkx /tmp/tmp.Zx1otCBoDv ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 11 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tUeTU9bmUM +++ mktemp ++ local LAST_ERR=/tmp/tmp.bHaw9B3d3n ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tUeTU9bmUM ++ cat /tmp/tmp.bHaw9B3d3n ++ rm /tmp/tmp.tUeTU9bmUM /tmp/tmp.bHaw9B3d3n ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 12 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.axkWnaymtc +++ mktemp ++ local LAST_ERR=/tmp/tmp.F0ixVueNLc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.axkWnaymtc ++ cat /tmp/tmp.F0ixVueNLc ++ rm /tmp/tmp.axkWnaymtc /tmp/tmp.F0ixVueNLc ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 13 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.THF2bg3fkk +++ mktemp ++ local LAST_ERR=/tmp/tmp.WwdxKG1Ftb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.THF2bg3fkk ++ cat /tmp/tmp.WwdxKG1Ftb ++ rm /tmp/tmp.THF2bg3fkk /tmp/tmp.WwdxKG1Ftb ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 14 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SgHvpJXzIM +++ mktemp ++ local LAST_ERR=/tmp/tmp.So1yyrIBri ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SgHvpJXzIM ++ cat /tmp/tmp.So1yyrIBri ++ rm /tmp/tmp.SgHvpJXzIM /tmp/tmp.So1yyrIBri ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + [[ 15 -ge 300 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qRE0tkWUmY +++ mktemp ++ local LAST_ERR=/tmp/tmp.W1RZ4Wzmrm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qRE0tkWUmY ++ cat /tmp/tmp.W1RZ4Wzmrm ++ rm /tmp/tmp.qRE0tkWUmY /tmp/tmp.W1RZ4Wzmrm ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4qkRJQ1Ano +++ mktemp ++ local LAST_ERR=/tmp/tmp.05PwcXHxpj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4qkRJQ1Ano ++ cat /tmp/tmp.05PwcXHxpj ++ rm /tmp/tmp.4qkRJQ1Ano /tmp/tmp.05PwcXHxpj ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.ZnBVMmbM0j ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.3WejD6jiyk +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.ZnBVMmbM0j +++++ cat /tmp/tmp.3WejD6jiyk +++++ rm /tmp/tmp.ZnBVMmbM0j /tmp/tmp.3WejD6jiyk +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.W4kEqs2ggS +++ mktemp ++ local LAST_ERR=/tmp/tmp.qMMW5hgVvJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.W4kEqs2ggS ++ cat /tmp/tmp.qMMW5hgVvJ ++ rm /tmp/tmp.W4kEqs2ggS /tmp/tmp.qMMW5hgVvJ ++ return 0 + [[ 2 == \2 ]] + echo ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.09qDqWC7XM +++ mktemp ++ local LAST_ERR=/tmp/tmp.vQE2RmEBDr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.09qDqWC7XM ++ cat /tmp/tmp.vQE2RmEBDr ++ rm /tmp/tmp.09qDqWC7XM /tmp/tmp.vQE2RmEBDr ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hxQVhGOTII +++ mktemp ++ local LAST_ERR=/tmp/tmp.kLdPtFCBrT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hxQVhGOTII ++ cat /tmp/tmp.kLdPtFCBrT ++ rm /tmp/tmp.hxQVhGOTII /tmp/tmp.kLdPtFCBrT ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ep1YJjaT25 +++ mktemp ++ local LAST_ERR=/tmp/tmp.xDi7BMGHeH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ep1YJjaT25 ++ cat /tmp/tmp.xDi7BMGHeH ++ rm /tmp/tmp.Ep1YJjaT25 /tmp/tmp.xDi7BMGHeH ++ return 0 + client_pod=pxc-client-58956d9c99-8ltqs + wait_pod pxc-client-58956d9c99-8ltqs + local pod=pxc-client-58956d9c99-8ltqs + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-8ltqs ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-8ltqs condition met waiting for pod/pxc-client-58956d9c99-8ltqs to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.Kmewg0QhXK/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.Kmewg0QhXK/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.Kmewg0QhXK/select-1.sql + kubectl_bin delete -f /tmp/tmp.Kmewg0QhXK/smart-update-version-service-reachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.O4wlmGjLn9 ++ mktemp + local LAST_ERR=/tmp/tmp.B7nqvs3vI3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.Kmewg0QhXK/smart-update-version-service-reachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.O4wlmGjLn9 perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update1-11246 namespace + cat /tmp/tmp.B7nqvs3vI3 + rm /tmp/tmp.O4wlmGjLn9 /tmp/tmp.B7nqvs3vI3 + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.lhNCRuzgkP ++ mktemp + local LAST_ERR=/tmp/tmp.qrQuEmSkKe + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lhNCRuzgkP persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update1-11246 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update1-11246 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update1-11246 namespace + cat /tmp/tmp.qrQuEmSkKe + rm /tmp/tmp.lhNCRuzgkP /tmp/tmp.qrQuEmSkKe + return 0 + desc cleanup + set +o xtrace ----------------------------------------------------------------------------------- cleanup ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.3lz0b3uysa ++ mktemp + local LAST_ERR=/tmp/tmp.fYcaqXjRgU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update1/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3lz0b3uysa deployment.apps "version-service" deleted from smart-update1-11246 namespace service "version-service" deleted from smart-update1-11246 namespace + cat /tmp/tmp.fYcaqXjRgU + rm /tmp/tmp.3lz0b3uysa /tmp/tmp.fYcaqXjRgU + return 0 + destroy smart-update1-11246 + local namespace=smart-update1-11246 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v level=info + sort -u + tee /tmp/tmp.Kmewg0QhXK/operator.log + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'the object has been modified' + grep -v 'get backup status: Job.batch' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.jPypdCFXCY +++ mktemp ++ local LAST_ERR=/tmp/tmp.WidmD2ivaL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jPypdCFXCY ++ cat /tmp/tmp.WidmD2ivaL ++ rm /tmp/tmp.jPypdCFXCY /tmp/tmp.WidmD2ivaL ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-665ff7485-wpndd ++ mktemp + local LAST_OUT=/tmp/tmp.Q5pL4jxjOd ++ mktemp + local LAST_ERR=/tmp/tmp.kGsDffaTak + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-665ff7485-wpndd + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Q5pL4jxjOd + cat /tmp/tmp.kGsDffaTak + rm /tmp/tmp.Q5pL4jxjOd /tmp/tmp.kGsDffaTak + return 0 } }, }, { }, }, { }, }, }, ""), { }, - }, - { - }, + }, ... // 16 identical fields ... // 16 identical fields 2026-04-09T04:15:01.835Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-04-09T04:15:01.835Z INFO setup Manager starting up {"gitCommit": "3b65d7fe6a0a5db845238d34252d90db1f886e1d", "gitBranch": "PR-2434-3b65d7fe", "buildTime": "2026-04-09T01:11:10Z", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"} 2026-04-09T04:15:01.835Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.32.13-gke.1205000"} 2026-04-09T04:15:01.839Z INFO setup Registering Components. 2026-04-09T04:15:02.264Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-04-09T04:15:02.265Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-04-09T04:15:02.265Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-04-09T04:15:02.265Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-04-09T04:15:02.265Z INFO controller-runtime.metrics Starting metrics server 2026-04-09T04:15:02.265Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-04-09T04:15:02.265Z INFO controller-runtime.webhook Starting webhook server 2026-04-09T04:15:02.265Z INFO setup Starting the Cmd. 2026-04-09T04:15:02.265Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-04-09T04:15:02.366Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-04-09T04:15:02.393Z DEBUG events percona-xtradb-cluster-operator-665ff7485-wpndd_7d479929-1d05-4d8e-bf20-c22fc717a70c became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"9f7f704e-056d-41fa-af19-5b8edd997f08","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1775708102387567009"}, "reason": "LeaderElection"} 2026-04-09T04:15:02.393Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-04-09T04:15:02.393Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-04-09T04:15:02.393Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-04-09T04:15:02.393Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-04-09T04:15:02.394Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-04-09T04:15:02.493Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-04-09T04:15:02.494Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-04-09T04:15:02.494Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-04-09T04:15:02.494Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-04-09T04:15:02.495Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-04-09T04:15:02.495Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-04-09T04:17:49.262Z INFO spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`. {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "1c977208-dfa5-4eb1-b195-6f37c6b01d8a"} 2026-04-09T04:17:52.322Z INFO spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`. {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "1c977208-dfa5-4eb1-b195-6f37c6b01d8a"} 2026-04-09T04:17:52.358Z INFO spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`. {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "1c977208-dfa5-4eb1-b195-6f37c6b01d8a"} 2026-04-09T04:17:55.459Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "1c977208-dfa5-4eb1-b195-6f37c6b01d8a", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-04-09T04:17:55.590Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "1c977208-dfa5-4eb1-b195-6f37c6b01d8a", "object": "smart-update-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-09T04:17:55.672Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "1c977208-dfa5-4eb1-b195-6f37c6b01d8a", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-09T04:17:55.720Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "1c977208-dfa5-4eb1-b195-6f37c6b01d8a", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:17:55.777Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "1c977208-dfa5-4eb1-b195-6f37c6b01d8a", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:17:55.843Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "1c977208-dfa5-4eb1-b195-6f37c6b01d8a", "object": "smart-update-proxysql", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:17:55.968Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "1c977208-dfa5-4eb1-b195-6f37c6b01d8a", "object": "smart-update-proxysql-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:17:56.886Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "ede9b539-81a2-445d-a5a8-daa6a50f5903", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-09T04:17:56.910Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "ede9b539-81a2-445d-a5a8-daa6a50f5903", "object": "smart-update-proxysql", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-09T04:19:22.875Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "593ec902-5ce7-4440-aec2-c5a58c54ae66", "user": "operator"} 2026-04-09T04:19:22.931Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "593ec902-5ce7-4440-aec2-c5a58c54ae66", "user": "monitor"} 2026-04-09T04:19:22.974Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "593ec902-5ce7-4440-aec2-c5a58c54ae66"} 2026-04-09T04:19:23.002Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "593ec902-5ce7-4440-aec2-c5a58c54ae66"} 2026-04-09T04:19:23.038Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "593ec902-5ce7-4440-aec2-c5a58c54ae66", "user": "xtrabackup"} 2026-04-09T04:19:23.078Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "593ec902-5ce7-4440-aec2-c5a58c54ae66"} 2026-04-09T04:19:23.106Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "593ec902-5ce7-4440-aec2-c5a58c54ae66", "user": "replication"} 2026-04-09T04:19:24.115Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "593ec902-5ce7-4440-aec2-c5a58c54ae66", "err": "get primary pxc pod: not found"} 2026-04-09T04:19:30.351Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "142f02d3-9d4f-4054-a06d-4235328bb72c", "err": "get primary pxc pod: not found"} 2026-04-09T04:19:36.492Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6467cbe7-9c03-42c4-8b82-d6e25cc04a6e", "err": "get primary pxc pod: not found"} 2026-04-09T04:22:01.940Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "46c0e367-341c-4d8f-aeb2-c0a547be75d7", "user": "root"} 2026-04-09T04:22:03.050Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "46c0e367-341c-4d8f-aeb2-c0a547be75d7", "new version": "8.4.5-5.1"} 2026-04-09T04:22:03.318Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "46c0e367-341c-4d8f-aeb2-c0a547be75d7"} 2026-04-09T04:22:04.902Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "abe1c1f6-7244-4103-8737-8f5cca2a4071"} 2026-04-09T04:22:11.204Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "47f46685-3e96-4227-a569-30c3e5bfc7aa"} 2026-04-09T04:22:17.486Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "d27d04fb-4c2f-407f-a889-0ba3c7b47144"} 2026-04-09T04:22:23.583Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "4e5a8e3a-1522-4ac7-be01-732663c07864"} 2026-04-09T04:22:30.060Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5e4cf2bf-edef-4240-8e32-338f091c9794"} 2026-04-09T04:22:36.248Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "72929f8b-b0b5-4d73-b4b9-7b49a009ee46"} 2026-04-09T04:22:42.501Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "f603ef6d-d16e-4c12-bac2-8553dd7227d9"} 2026-04-09T04:22:48.934Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ede6bcb-faec-4555-9c0c-8277f997f74b"} 2026-04-09T04:22:55.094Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "16cecbb2-ced8-41bd-921f-981cbe972b95"} 2026-04-09T04:23:01.134Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "23e007d6-489f-4446-adb1-fce4a5f4d60d"} 2026-04-09T04:23:07.594Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "97d1c1b4-abeb-490c-8a32-793d81e402ef"} 2026-04-09T04:23:10.216Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-04-09T04:23:10.275Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-04-09T04:23:11.390Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49"} 2026-04-09T04:23:11.395Z INFO apply changes to secondary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2"} 2026-04-09T04:23:11.395Z INFO primary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-0.smart-update-pxc.smart-update1-11246.svc.cluster.local"} 2026-04-09T04:23:11.712Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49"} 2026-04-09T04:23:12.114Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2", "currentRevision": "smart-update-pxc-5d5b84b669", "targetRevision": "smart-update-pxc-6946998cdd"} 2026-04-09T04:23:22.248Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2", "currentRevision": "smart-update-pxc-5d5b84b669", "targetRevision": "smart-update-pxc-6946998cdd"} 2026-04-09T04:23:32.205Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2", "phase": "Pending"} 2026-04-09T04:23:32.205Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2", "reason": "PodInitializing"} 2026-04-09T04:23:42.197Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2", "phase": "Pending"} 2026-04-09T04:23:42.197Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2", "reason": "PodInitializing"} 2026-04-09T04:24:22.239Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2"} 2026-04-09T04:24:22.257Z INFO waiting for pod to be present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2", "host": "smart-update-pxc-2.smart-update-pxc.smart-update1-11246", "hostgroup": "10"} 2026-04-09T04:24:32.259Z INFO waiting for pod to be present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2", "host": "smart-update-pxc-2.smart-update-pxc.smart-update1-11246", "hostgroup": "10"} 2026-04-09T04:24:42.259Z INFO waiting for pod to be present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2", "host": "smart-update-pxc-2.smart-update-pxc.smart-update1-11246", "hostgroup": "10"} 2026-04-09T04:24:52.259Z INFO waiting for pod to be present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2", "host": "smart-update-pxc-2.smart-update-pxc.smart-update1-11246", "hostgroup": "10"} 2026-04-09T04:25:02.259Z INFO waiting for pod to be present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2", "host": "smart-update-pxc-2.smart-update-pxc.smart-update1-11246", "hostgroup": "10"} 2026-04-09T04:25:12.259Z INFO pod present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2", "host": "smart-update-pxc-2.smart-update-pxc.smart-update1-11246", "hostgroup": "10"} 2026-04-09T04:25:12.263Z INFO apply changes to secondary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-1"} 2026-04-09T04:25:12.263Z INFO pod is online {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-2"} 2026-04-09T04:25:12.453Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-1", "currentRevision": "smart-update-pxc-5d5b84b669", "targetRevision": "smart-update-pxc-6946998cdd"} 2026-04-09T04:25:22.565Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-1", "currentRevision": "smart-update-pxc-5d5b84b669", "targetRevision": "smart-update-pxc-6946998cdd"} 2026-04-09T04:25:32.540Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-1", "phase": "Pending"} 2026-04-09T04:25:32.540Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-1", "reason": "PodInitializing"} 2026-04-09T04:26:12.576Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-1"} 2026-04-09T04:26:12.592Z INFO pod present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-1", "host": "smart-update-pxc-1.smart-update-pxc.smart-update1-11246", "hostgroup": "10"} 2026-04-09T04:26:12.596Z INFO apply changes to primary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-0"} 2026-04-09T04:26:12.596Z INFO pod is online {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-1"} 2026-04-09T04:26:12.795Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-0", "currentRevision": "smart-update-pxc-5d5b84b669", "targetRevision": "smart-update-pxc-6946998cdd"} 2026-04-09T04:26:22.913Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-0", "currentRevision": "smart-update-pxc-5d5b84b669", "targetRevision": "smart-update-pxc-6946998cdd"} 2026-04-09T04:26:32.879Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-0", "phase": "Pending"} 2026-04-09T04:26:32.879Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-0", "reason": "PodInitializing"} 2026-04-09T04:26:42.881Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-0", "phase": "Pending"} 2026-04-09T04:26:42.881Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-0", "reason": "PodInitializing"} 2026-04-09T04:27:22.909Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-0"} 2026-04-09T04:27:22.928Z INFO pod present in hostgroup {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-0", "host": "smart-update-pxc-0.smart-update-pxc.smart-update1-11246", "hostgroup": "11"} 2026-04-09T04:27:22.931Z INFO pod is online {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "pod": "smart-update-pxc-0"} 2026-04-09T04:27:22.931Z INFO smart update finished {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49"} 2026-04-09T04:27:22.983Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "6ff7b6d8-561d-4884-8f79-3305ed59df49", "new version": "8.4.7-7.1"} 2026-04-09T04:27:25.042Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "7e746920-a0f7-4cbd-bf27-baec9f086f54"} 2026-04-09T04:27:31.335Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "415a7c0c-1503-48f5-b5ff-97509f336728"} 2026-04-09T04:27:38.018Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "a35343e4-60fd-4738-a03f-38704c20fd54"} 2026-04-09T04:27:43.883Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "8ca76714-8b51-435c-86c4-84279024d96c"} 2026-04-09T04:27:50.732Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "d64a8212-842b-48b7-be1e-2815d45b5a15"} 2026-04-09T04:28:31.203Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "ccbd6f70-b6f2-43bc-af88-342279bf022b", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2026-04-09T04:28:47.487Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "c7195bdb-ab2d-45d6-b495-e495dc3811f4", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-04-09T04:28:47.524Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "c7195bdb-ab2d-45d6-b495-e495dc3811f4", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-04-09T04:28:48.131Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "c7195bdb-ab2d-45d6-b495-e495dc3811f4", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:92\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:25\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:27\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:50\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:543\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:556\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} 2026-04-09T04:28:48.241Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "db5023e1-da9e-4682-aab4-a318f6be36d5", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-09T04:28:48.288Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "db5023e1-da9e-4682-aab4-a318f6be36d5", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-09T04:28:48.332Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "db5023e1-da9e-4682-aab4-a318f6be36d5", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:28:48.378Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "db5023e1-da9e-4682-aab4-a318f6be36d5", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:28:48.450Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "db5023e1-da9e-4682-aab4-a318f6be36d5", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:28:48.526Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "db5023e1-da9e-4682-aab4-a318f6be36d5", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:28:49.334Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "f2753612-ac3f-446a-a382-9c57b79bcb54", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-09T04:28:49.357Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "f2753612-ac3f-446a-a382-9c57b79bcb54", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-09T04:32:44.072Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "695e34e5-fd9e-45dd-8a0a-2658ddf44fa6", "err": "failed to ensure cluster readonly status: connect to pod smart-update-pxc-2: dial tcp: lookup smart-update-pxc-2.smart-update-pxc.smart-update1-11246 on 34.118.224.10:53: no such host"} 2026-04-09T04:32:46.263Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "a7467d19-49d7-488d-9d72-c3c8b5e3568d", "new version": "8.4.5-5.1"} 2026-04-09T04:34:17.329Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-04-09T04:34:17.386Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-04-09T04:34:18.475Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866"} 2026-04-09T04:34:18.481Z INFO apply changes to secondary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-2"} 2026-04-09T04:34:18.481Z INFO primary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-0.smart-update-pxc.smart-update1-11246"} 2026-04-09T04:34:19.161Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-2", "currentRevision": "smart-update-pxc-5778b47548", "targetRevision": "smart-update-pxc-756b8b8689"} 2026-04-09T04:34:29.450Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-2", "currentRevision": "smart-update-pxc-5778b47548", "targetRevision": "smart-update-pxc-756b8b8689"} 2026-04-09T04:34:39.257Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-2", "phase": "Pending"} 2026-04-09T04:34:39.257Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-2", "reason": "PodInitializing"} 2026-04-09T04:35:19.320Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-2"} 2026-04-09T04:35:19.332Z INFO apply changes to secondary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-1"} 2026-04-09T04:35:19.515Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-1", "currentRevision": "smart-update-pxc-5778b47548", "targetRevision": "smart-update-pxc-756b8b8689"} 2026-04-09T04:35:29.645Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-1", "currentRevision": "smart-update-pxc-5778b47548", "targetRevision": "smart-update-pxc-756b8b8689"} 2026-04-09T04:35:39.609Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-1", "phase": "Pending"} 2026-04-09T04:35:39.609Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-1", "reason": "PodInitializing"} 2026-04-09T04:36:19.634Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-1"} 2026-04-09T04:36:19.645Z INFO apply changes to primary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-0"} 2026-04-09T04:36:19.839Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-0", "currentRevision": "smart-update-pxc-5778b47548", "targetRevision": "smart-update-pxc-756b8b8689"} 2026-04-09T04:36:29.971Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-0", "currentRevision": "smart-update-pxc-5778b47548", "targetRevision": "smart-update-pxc-756b8b8689"} 2026-04-09T04:37:20.064Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "pod": "smart-update-pxc-0"} 2026-04-09T04:37:20.074Z INFO smart update finished {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866"} 2026-04-09T04:37:20.304Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "5ef9653a-3ce5-479a-aa49-f66e3417b866", "new version": "8.4.7-7.1"} 2026-04-09T04:38:22.125Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "cb0510e9-80ba-4fca-8bf8-4c402cb16c98", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2026-04-09T04:38:37.127Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "3bed850f-1d30-44ea-b932-6d1cdb072e04", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-04-09T04:38:37.148Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "3bed850f-1d30-44ea-b932-6d1cdb072e04", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-04-09T04:38:37.735Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "3bed850f-1d30-44ea-b932-6d1cdb072e04", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:92\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:25\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:27\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:50\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:543\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:556\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} 2026-04-09T04:38:37.838Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "3e6dbfb3-b893-46d3-b71c-a43e06debe4e", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-09T04:38:37.884Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "3e6dbfb3-b893-46d3-b71c-a43e06debe4e", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-09T04:38:37.930Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "3e6dbfb3-b893-46d3-b71c-a43e06debe4e", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:38:37.977Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "3e6dbfb3-b893-46d3-b71c-a43e06debe4e", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:38:38.036Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "3e6dbfb3-b893-46d3-b71c-a43e06debe4e", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:38:38.116Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "3e6dbfb3-b893-46d3-b71c-a43e06debe4e", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:38:38.966Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "a57bd0b3-362c-4f93-8389-cb2fbf770119", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-09T04:38:39.000Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "a57bd0b3-362c-4f93-8389-cb2fbf770119", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-09T04:42:41.630Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "bbc38ac9-2670-4b2a-a6b2-df0a97e57a47", "new version": "8.4.5-5.1"} 2026-04-09T04:44:08.343Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-04-09T04:44:08.451Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-04-09T04:44:09.527Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d"} 2026-04-09T04:44:09.532Z INFO apply changes to secondary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-2"} 2026-04-09T04:44:09.532Z INFO primary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-0.smart-update-pxc.smart-update1-11246"} 2026-04-09T04:44:10.209Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-2", "currentRevision": "smart-update-pxc-86bdb8fd67", "targetRevision": "smart-update-pxc-7df5c7647d"} 2026-04-09T04:44:20.350Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-2", "currentRevision": "smart-update-pxc-86bdb8fd67", "targetRevision": "smart-update-pxc-7df5c7647d"} 2026-04-09T04:45:00.361Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-2"} 2026-04-09T04:45:00.374Z INFO apply changes to secondary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-1"} 2026-04-09T04:45:00.564Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-1", "currentRevision": "smart-update-pxc-86bdb8fd67", "targetRevision": "smart-update-pxc-7df5c7647d"} 2026-04-09T04:45:10.681Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-1", "currentRevision": "smart-update-pxc-86bdb8fd67", "targetRevision": "smart-update-pxc-7df5c7647d"} 2026-04-09T04:45:20.671Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-1", "phase": "Pending"} 2026-04-09T04:45:20.671Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-1", "reason": "PodInitializing"} 2026-04-09T04:46:00.680Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-1"} 2026-04-09T04:46:00.695Z INFO apply changes to primary pod {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-0"} 2026-04-09T04:46:00.996Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-0", "currentRevision": "smart-update-pxc-86bdb8fd67", "targetRevision": "smart-update-pxc-7df5c7647d"} 2026-04-09T04:46:11.109Z INFO Pod is not updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-0", "currentRevision": "smart-update-pxc-86bdb8fd67", "targetRevision": "smart-update-pxc-7df5c7647d"} 2026-04-09T04:46:21.084Z INFO Pod is not running {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-0", "phase": "Pending"} 2026-04-09T04:46:21.084Z INFO pod is waiting {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-0", "reason": "PodInitializing"} 2026-04-09T04:47:01.193Z INFO Pod is updated, running and ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "pod": "smart-update-pxc-0"} 2026-04-09T04:47:01.206Z INFO smart update finished {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d"} 2026-04-09T04:47:01.297Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "93d9fac6-372b-48e5-aacf-bcb698c0a96d", "new version": "8.4.7-7.1"} 2026-04-09T04:47:54.553Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update1-11246"}, "namespace": "smart-update1-11246", "name": "smart-update", "reconcileID": "e1e3093c-e61a-4ecb-986c-bde9c45355b5", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} ... // 22 identical fields ... // 2 identical fields ... // 3 identical fields ... // 3 identical fields ... // 3 identical fields ... // 4 identical fields ... // 5 identical fields ... // 6 identical fields ... // 7 identical fields - ":8.4.5-5.1", ... // 9 identical fields AccessModes: nil, ActiveDeadlineSeconds: nil, Affinity: &{PodAntiAffinity: &{RequiredDuringSchedulingIgnoredDuringExecution: {{LabelSelector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "smart-update", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, TopologyKey: "kubernetes.io/hostname"}}}}, - Annotations: map[string]string{ + Annotations: map[string]string{ - APIVersion: "apps/v1", - APIVersion: "apps/v1", Args: {"mysqld"}, AutomountServiceAccountToken: nil, + AvailableReplicas: 0, - AvailableReplicas: 3, AWSElasticBlockStore: nil, AzureFile: nil, Capacity: nil, - CollisionCount: &0, + CollisionCount: nil, Command: {"/var/lib/mysql/pxc-entrypoint.sh"}, Conditions: nil, ConfigMap: &v1.ConfigMapVolumeSource{ ContainerPort: 3306, ContainerPort: 33060, ContainerPort: 33062, ContainerPort: 4444, ContainerPort: 4567, ContainerPort: 4568, Containers: []v1.Container{ + CreationTimestamp: v1.Time{}, - CreationTimestamp: v1.Time{Time: s"2026-04-09 04:17:55 +0000 UTC"}, - CreationTimestamp: v1.Time{Time: s"2026-04-09 04:28:48 +0000 UTC"}, - CreationTimestamp: v1.Time{Time: s"2026-04-09 04:38:37 +0000 UTC"}, + CurrentReplicas: 0, - CurrentReplicas: 3, + CurrentRevision: "", - CurrentRevision: "smart-update-pxc-5778b47548", - CurrentRevision: "smart-update-pxc-5d5b84b669", - CurrentRevision: "smart-update-pxc-86bdb8fd67", DataSource: nil, DataSourceRef: nil, - DefaultMode: &420, - DefaultMode: &420, + DefaultMode: nil, + DefaultMode: nil, DeletionGracePeriodSeconds: nil, DeletionTimestamp: nil, + DeprecatedServiceAccount: "", - DeprecatedServiceAccount: "default", + DNSPolicy: "", - DNSPolicy: "ClusterFirst", EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "smart-update-env-vars-pxc"}, Optional: &true}}}, Env: {{Name: "PXC_SERVICE", Value: "smart-update-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-smart-update"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-smart-update"}, Key: "xtrabackup"}}}, ...}, EphemeralContainers: nil, FailureThreshold: 3, FC: nil, - FieldsType: "FieldsV1", - FieldsType: "FieldsV1", - FieldsV1: s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`..., - FieldsV1: s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`..., Finalizers: nil, + Generation: 0, - Generation: 1, github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1 GitRepo: nil, /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:495 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:259 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:484 HostAliases: nil, HostIP: "", HostPort: 0, ImagePullPolicy: "Always", Image: strings.Join({ InitContainers: []v1.Container{ InitialDelaySeconds: 300, ISCSI: nil, Items: nil, Items: nil, + "lab", Labels: nil, Lifecycle: nil, LivenessProbe: &v1.Probe{ LocalObjectReference: {Name: "auto-smart-update-pxc"}, LocalObjectReference: {Name: "smart-update-pxc"}, + ManagedFields: nil, - ManagedFields: []v1.ManagedFieldsEntry{ - Manager: "kube-controller-manager", - Manager: "percona-xtradb-cluster-operator", MinReadySeconds: 0, Name: "auto-config", Name: "config", Name: "ist", Name: "mysql", Name: "mysql-admin", Name: "mysql-init-file", Name: "mysql-users-secret-file", Name: "mysqlx", Name: "pxc", Namespace: "smart-update1-11246", Name: "ssl", Name: "ssl-internal", Name: "sst", {Name: "tmp", VolumeSource: {EmptyDir: &{}}}, Name: "vault-keyring-secret", Name: "write-set", NFS: nil, NodeName: "", NodeSelector: nil, ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "smart-update", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "7580631e733000666f54b01b98d918f7", "percona.com/ssl-internal-hash": "235671901cfbcec140826e2d88c26bbb"}}, ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "smart-update", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, ObjectMeta: v1.ObjectMeta{ + ObservedGeneration: 0, - ObservedGeneration: 1, - Operation: "Update", - Operation: "Update", + "-operator:main-pxc8.4", Optional: &false, Optional: &true, Optional: &true, Ordinals: nil, OS: nil, Overhead: nil, OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "smart-update", UID: "2d42311f-7444-40c7-8f69-fd73728258a8", ...}}, OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "smart-update", UID: "baaecc38-5220-4e1a-b576-a7ba5e767d23", ...}}, OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "smart-update", UID: "d879bec0-13b1-4375-93fc-ba1cf96f72bd", ...}}, "percona", + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzbWFydC11cGRhdGUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifX0sInRlbXBsYXRlIjp7Im1ldGFkYXRhIjp7ImxhYmVscyI6eyJhcHAua3ViZXJuZXRlcy5pby9jb21wb25lbnQiOiJweGMiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6InNtYXJ0LXVwZGF0ZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9LCJhbm5vdGF0aW9ucyI6eyJrdWJlY3RsLmt1YmVybmV0ZXMuaW8vZGVmYXVsdC1jb250YWluZXIiOiJweGMiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiNzU4MDYzMWU3MzMwMDA2NjZmNTRiMDFiOThkOTE4ZjciLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6IjIzNTY3MTkwMWNmYmNlYzE0MDgyNmUyZDg4YzI2YmJiIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJ0bXAiLCJlbXB0eURpciI6e319LHsibmFtZSI6ImNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoic21hcnQtdXBkYXRlLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtc3NsLWludGVybmFsIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJzc2wiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6InNtYXJ0LXVwZGF0ZS1zc2wiLCJvcHRpb25hbCI6ZmFsc2V9fSx7Im5hbWUiOiJhdXRvLWNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoiYXV0by1zbWFydC11cGRhdGUtcHhjIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJ2YXVsdC1rZXlyaW5nLXNlY3JldCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoic21hcnQtdXBkYXRlLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXlzcWwtaW5pdC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtbXlzcWwtaW5pdCIsIm9wdGlvbmFsIjp0cnVlfX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjQzNC0zYjY1ZDdmZSIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImRhdGFkaXIiLCJtb3VudFBhdGgiOiIvdmFyL2xpYi9teXNxbCJ9XSwiaW1hZ2VQdWxsUG9saWN5IjoiQWx3YXlzIn1dLCJjb250YWluZXJzIjpbeyJuYW1lIjoicHhjIiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6bWFpbi1weGM4LjQiLCJjb21tYW5kIjpbIi92YXIvbGliL215c3FsL3B4Yy1lbnRyeXBvaW50LnNoIl0sImFyZ3MiOlsibXlzcWxkIl0sInBvcnRzIjpbeyJuYW1lIjoibXlzcWwiLCJjb250YWluZXJQb3J0IjozMzA2fSx7Im5hbWUiOiJzc3QiLCJjb250YWluZXJQb3J0Ijo0NDQ0fSx7Im5hbWUiOiJ3cml0ZS1zZXQiLCJjb250YWluZXJQb3J0Ijo0NTY3fSx7Im5hbWUiOiJpc3QiLCJjb250YWluZXJQb3J0Ijo0NTY4fSx7Im5hbWUiOiJteXNxbC1hZG1pbiIsImNvbnRhaW5lclBvcnQiOjMzMDYyfSx7Im5hbWUiOiJteXNxbHgiLCJjb250YWluZXJQb3J0IjozMzA2MH1dLCJlbnZGcm9tIjpbeyJzZWNyZXRSZWYiOnsibmFtZSI6InNtYXJ0LXVwZGF0ZS1lbnYtdmFycy1weGMiLCJvcHRpb25hbCI6dHJ1ZX19XSwiZW52IjpbeyJuYW1lIjoiUFhDX1NFUlZJQ0UiLCJ2YWx1ZSI6InNtYXJ0LXVwZGF0ZS1weGMtdW5yZWFkeSJ9LHsibmFtZSI6Ik1PTklUT1JfSE9TVCIsInZhbHVlIjoiJSJ9LHsibmFtZSI6Ik1ZU1FMX1JPT1RfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJyb290In19fSx7Im5hbWUiOiJYVFJBQkFDS1VQX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5IjoieHRyYWJhY2t1cCJ9fX0seyJuYW1lIjoiTU9OSVRPUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6Im1vbml0b3IifX19LHsibmFtZSI6IkNMVVNURVJfSEFTSCIsInZhbHVlIjoiMjU4MDA0MCJ9LHsibmFtZSI6Ik9QRVJBVE9SX0FETUlOX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5Ijoib3BlcmF0b3IifX19LHsibmFtZSI6IkxJVkVORVNTX0NIRUNLX1RJTUVPVVQiLCJ2YWx1ZSI6IjUifSx7Im5hbWUiOiJSRUFESU5FU1NfQ0hFQ0tfVElNRU9VVCIsInZhbHVlIjoiMTUifSx7Im5hbWUiOiJERUZBVUxUX0FVVEhFTlRJQ0FUSU9OX1BMVUdJTiIsInZhbHVlIjoiY2FjaGluZ19zaGEyX3Bhc3N3b3JkIn0seyJuYW1lIjoiTVlTUUxfTk9USUZZX1NPQ0tFVCIsInZhbHVlIjoiL3Zhci9saWIvbXlzcWwvbm90"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzbWFydC11cGRhdGUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifX0sInRlbXBsYXRlIjp7Im1ldGFkYXRhIjp7ImxhYmVscyI6eyJhcHAua3ViZXJuZXRlcy5pby9jb21wb25lbnQiOiJweGMiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6InNtYXJ0LXVwZGF0ZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9LCJhbm5vdGF0aW9ucyI6eyJrdWJlY3RsLmt1YmVybmV0ZXMuaW8vZGVmYXVsdC1jb250YWluZXIiOiJweGMiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiNzU4MDYzMWU3MzMwMDA2NjZmNTRiMDFiOThkOTE4ZjciLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6IjIzNTY3MTkwMWNmYmNlYzE0MDgyNmUyZDg4YzI2YmJiIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJ0bXAiLCJlbXB0eURpciI6e319LHsibmFtZSI6ImNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoic21hcnQtdXBkYXRlLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtc3NsLWludGVybmFsIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJzc2wiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6InNtYXJ0LXVwZGF0ZS1zc2wiLCJvcHRpb25hbCI6ZmFsc2V9fSx7Im5hbWUiOiJhdXRvLWNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoiYXV0by1zbWFydC11cGRhdGUtcHhjIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJ2YXVsdC1rZXlyaW5nLXNlY3JldCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoic21hcnQtdXBkYXRlLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXlzcWwtaW5pdC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtbXlzcWwtaW5pdCIsIm9wdGlvbmFsIjp0cnVlfX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjQzNC0zYjY1ZDdmZSIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImRhdGFkaXIiLCJtb3VudFBhdGgiOiIvdmFyL2xpYi9teXNxbCJ9XSwiaW1hZ2VQdWxsUG9saWN5IjoiQWx3YXlzIn1dLCJjb250YWluZXJzIjpbeyJuYW1lIjoicHhjIiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6bWFpbi1weGM4LjQiLCJjb21tYW5kIjpbIi92YXIvbGliL215c3FsL3B4Yy1lbnRyeXBvaW50LnNoIl0sImFyZ3MiOlsibXlzcWxkIl0sInBvcnRzIjpbeyJuYW1lIjoibXlzcWwiLCJjb250YWluZXJQb3J0IjozMzA2fSx7Im5hbWUiOiJzc3QiLCJjb250YWluZXJQb3J0Ijo0NDQ0fSx7Im5hbWUiOiJ3cml0ZS1zZXQiLCJjb250YWluZXJQb3J0Ijo0NTY3fSx7Im5hbWUiOiJpc3QiLCJjb250YWluZXJQb3J0Ijo0NTY4fSx7Im5hbWUiOiJteXNxbC1hZG1pbiIsImNvbnRhaW5lclBvcnQiOjMzMDYyfSx7Im5hbWUiOiJteXNxbHgiLCJjb250YWluZXJQb3J0IjozMzA2MH1dLCJlbnZGcm9tIjpbeyJzZWNyZXRSZWYiOnsibmFtZSI6InNtYXJ0LXVwZGF0ZS1lbnYtdmFycy1weGMiLCJvcHRpb25hbCI6dHJ1ZX19XSwiZW52IjpbeyJuYW1lIjoiUFhDX1NFUlZJQ0UiLCJ2YWx1ZSI6InNtYXJ0LXVwZGF0ZS1weGMtdW5yZWFkeSJ9LHsibmFtZSI6Ik1PTklUT1JfSE9TVCIsInZhbHVlIjoiJSJ9LHsibmFtZSI6Ik1ZU1FMX1JPT1RfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJyb290In19fSx7Im5hbWUiOiJYVFJBQkFDS1VQX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5IjoieHRyYWJhY2t1cCJ9fX0seyJuYW1lIjoiTU9OSVRPUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6Im1vbml0b3IifX19LHsibmFtZSI6IkNMVVNURVJfSEFTSCIsInZhbHVlIjoiMTUwMTAyOCJ9LHsibmFtZSI6Ik9QRVJBVE9SX0FETUlOX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5Ijoib3BlcmF0b3IifX19LHsibmFtZSI6IkxJVkVORVNTX0NIRUNLX1RJTUVPVVQiLCJ2YWx1ZSI6IjUifSx7Im5hbWUiOiJSRUFESU5FU1NfQ0hFQ0tfVElNRU9VVCIsInZhbHVlIjoiMTUifSx7Im5hbWUiOiJERUZBVUxUX0FVVEhFTlRJQ0FUSU9OX1BMVUdJTiIsInZhbHVlIjoiY2FjaGluZ19zaGEyX3Bhc3N3b3JkIn0seyJuYW1lIjoiTVlTUUxfTk9USUZZX1NPQ0tFVCIsInZhbHVlIjoiL3Zhci9saWIvbXlzcWwvbm90"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzbWFydC11cGRhdGUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifX0sInRlbXBsYXRlIjp7Im1ldGFkYXRhIjp7ImxhYmVscyI6eyJhcHAua3ViZXJuZXRlcy5pby9jb21wb25lbnQiOiJweGMiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6InNtYXJ0LXVwZGF0ZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9LCJhbm5vdGF0aW9ucyI6eyJrdWJlY3RsLmt1YmVybmV0ZXMuaW8vZGVmYXVsdC1jb250YWluZXIiOiJweGMiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiNzU4MDYzMWU3MzMwMDA2NjZmNTRiMDFiOThkOTE4ZjciLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6IjIzNTY3MTkwMWNmYmNlYzE0MDgyNmUyZDg4YzI2YmJiIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJ0bXAiLCJlbXB0eURpciI6e319LHsibmFtZSI6ImNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoic21hcnQtdXBkYXRlLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtc3NsLWludGVybmFsIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJzc2wiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6InNtYXJ0LXVwZGF0ZS1zc2wiLCJvcHRpb25hbCI6ZmFsc2V9fSx7Im5hbWUiOiJhdXRvLWNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoiYXV0by1zbWFydC11cGRhdGUtcHhjIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJ2YXVsdC1rZXlyaW5nLXNlY3JldCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoic21hcnQtdXBkYXRlLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXlzcWwtaW5pdC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtbXlzcWwtaW5pdCIsIm9wdGlvbmFsIjp0cnVlfX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjQzNC0zYjY1ZDdmZSIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImRhdGFkaXIiLCJtb3VudFBhdGgiOiIvdmFyL2xpYi9teXNxbCJ9XSwiaW1hZ2VQdWxsUG9saWN5IjoiQWx3YXlzIn1dLCJjb250YWluZXJzIjpbeyJuYW1lIjoicHhjIiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6bWFpbi1weGM4LjQiLCJjb21tYW5kIjpbIi92YXIvbGliL215c3FsL3B4Yy1lbnRyeXBvaW50LnNoIl0sImFyZ3MiOlsibXlzcWxkIl0sInBvcnRzIjpbeyJuYW1lIjoibXlzcWwiLCJjb250YWluZXJQb3J0IjozMzA2fSx7Im5hbWUiOiJzc3QiLCJjb250YWluZXJQb3J0Ijo0NDQ0fSx7Im5hbWUiOiJ3cml0ZS1zZXQiLCJjb250YWluZXJQb3J0Ijo0NTY3fSx7Im5hbWUiOiJpc3QiLCJjb250YWluZXJQb3J0Ijo0NTY4fSx7Im5hbWUiOiJteXNxbC1hZG1pbiIsImNvbnRhaW5lclBvcnQiOjMzMDYyfSx7Im5hbWUiOiJteXNxbHgiLCJjb250YWluZXJQb3J0IjozMzA2MH1dLCJlbnZGcm9tIjpbeyJzZWNyZXRSZWYiOnsibmFtZSI6InNtYXJ0LXVwZGF0ZS1lbnYtdmFycy1weGMiLCJvcHRpb25hbCI6dHJ1ZX19XSwiZW52IjpbeyJuYW1lIjoiUFhDX1NFUlZJQ0UiLCJ2YWx1ZSI6InNtYXJ0LXVwZGF0ZS1weGMtdW5yZWFkeSJ9LHsibmFtZSI6Ik1PTklUT1JfSE9TVCIsInZhbHVlIjoiJSJ9LHsibmFtZSI6Ik1ZU1FMX1JPT1RfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJyb290In19fSx7Im5hbWUiOiJYVFJBQkFDS1VQX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5IjoieHRyYWJhY2t1cCJ9fX0seyJuYW1lIjoiTU9OSVRPUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6Im1vbml0b3IifX19LHsibmFtZSI6IkNMVVNURVJfSEFTSCIsInZhbHVlIjoiNDA2NTkyOCJ9LHsibmFtZSI6Ik9QRVJBVE9SX0FETUlOX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5Ijoib3BlcmF0b3IifX19LHsibmFtZSI6IkxJVkVORVNTX0NIRUNLX1RJTUVPVVQiLCJ2YWx1ZSI6IjUifSx7Im5hbWUiOiJSRUFESU5FU1NfQ0hFQ0tfVElNRU9VVCIsInZhbHVlIjoiMTUifSx7Im5hbWUiOiJERUZBVUxUX0FVVEhFTlRJQ0FUSU9OX1BMVUdJTiIsInZhbHVlIjoiY2FjaGluZ19zaGEyX3Bhc3N3b3JkIn0seyJuYW1lIjoiTVlTUUxfTk9USUZZX1NPQ0tFVCIsInZhbHVlIjoiL3Zhci9saWIvbXlzcWwvbm90"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzbWFydC11cGRhdGUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifX0sInRlbXBsYXRlIjp7Im1ldGFkYXRhIjp7ImxhYmVscyI6eyJhcHAua3ViZXJuZXRlcy5pby9jb21wb25lbnQiOiJweGMiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6InNtYXJ0LXVwZGF0ZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9LCJhbm5vdGF0aW9ucyI6eyJrdWJlY3RsLmt1YmVybmV0ZXMuaW8vZGVmYXVsdC1jb250YWluZXIiOiJweGMiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiNzU4MDYzMWU3MzMwMDA2NjZmNTRiMDFiOThkOTE4ZjciLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6IjIzNTY3MTkwMWNmYmNlYzE0MDgyNmUyZDg4YzI2YmJiIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJ0bXAiLCJlbXB0eURpciI6e319LHsibmFtZSI6ImNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoic21hcnQtdXBkYXRlLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtc3NsLWludGVybmFsIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJzc2wiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6InNtYXJ0LXVwZGF0ZS1zc2wiLCJvcHRpb25hbCI6ZmFsc2V9fSx7Im5hbWUiOiJhdXRvLWNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoiYXV0by1zbWFydC11cGRhdGUtcHhjIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJ2YXVsdC1rZXlyaW5nLXNlY3JldCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoic21hcnQtdXBkYXRlLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXlzcWwtaW5pdC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtbXlzcWwtaW5pdCIsIm9wdGlvbmFsIjp0cnVlfX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjQzNC0zYjY1ZDdmZSIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImRhdGFkaXIiLCJtb3VudFBhdGgiOiIvdmFyL2xpYi9teXNxbCJ9XSwiaW1hZ2VQdWxsUG9saWN5IjoiQWx3YXlzIn1dLCJjb250YWluZXJzIjpbeyJuYW1lIjoicHhjIiwiaW1hZ2UiOiJwZXJjb25hL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXI6OC40LjUtNS4xIiwiY29tbWFuZCI6WyIvdmFyL2xpYi9teXNxbC9weGMtZW50cnlwb2ludC5zaCJdLCJhcmdzIjpbIm15c3FsZCJdLCJwb3J0cyI6W3sibmFtZSI6Im15c3FsIiwiY29udGFpbmVyUG9ydCI6MzMwNn0seyJuYW1lIjoic3N0IiwiY29udGFpbmVyUG9ydCI6NDQ0NH0seyJuYW1lIjoid3JpdGUtc2V0IiwiY29udGFpbmVyUG9ydCI6NDU2N30seyJuYW1lIjoiaXN0IiwiY29udGFpbmVyUG9ydCI6NDU2OH0seyJuYW1lIjoibXlzcWwtYWRtaW4iLCJjb250YWluZXJQb3J0IjozMzA2Mn0seyJuYW1lIjoibXlzcWx4IiwiY29udGFpbmVyUG9ydCI6MzMwNjB9XSwiZW52RnJvbSI6W3sic2VjcmV0UmVmIjp7Im5hbWUiOiJzbWFydC11cGRhdGUtZW52LXZhcnMtcHhjIiwib3B0aW9uYWwiOnRydWV9fV0sImVudiI6W3sibmFtZSI6IlBYQ19TRVJWSUNFIiwidmFsdWUiOiJzbWFydC11cGRhdGUtcHhjLXVucmVhZHkifSx7Im5hbWUiOiJNT05JVE9SX0hPU1QiLCJ2YWx1ZSI6IiUifSx7Im5hbWUiOiJNWVNRTF9ST09UX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5Ijoicm9vdCJ9fX0seyJuYW1lIjoiWFRSQUJBQ0tVUF9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6Inh0cmFiYWNrdXAifX19LHsibmFtZSI6Ik1PTklUT1JfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJtb25pdG9yIn19fSx7Im5hbWUiOiJDTFVTVEVSX0hBU0giLCJ2YWx1ZSI6IjE1MDEwMjgifSx7Im5hbWUiOiJPUEVSQVRPUl9BRE1JTl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6Im9wZXJhdG9yIn19fSx7Im5hbWUiOiJMSVZFTkVTU19DSEVDS19USU1FT1VUIiwidmFsdWUiOiI1In0seyJuYW1lIjoiUkVBRElORVNTX0NIRUNLX1RJTUVPVVQiLCJ2YWx1ZSI6IjE1In0seyJuYW1lIjoiREVGQVVMVF9BVVRIRU5USUNBVElPTl9QTFVHSU4iLCJ2YWx1ZSI6ImNhY2hpbmdfc2hhMl9wYXNzd29yZCJ9LHsibmFtZSI6Ik1ZU1FMX05PVElGWV9TT0NLRVQiLCJ2YWx1ZSI6Ii92YXIvbGliL215c3FsL25vdGlmeS5zb2NrIn0seyJu"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzbWFydC11cGRhdGUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifX0sInRlbXBsYXRlIjp7Im1ldGFkYXRhIjp7ImxhYmVscyI6eyJhcHAua3ViZXJuZXRlcy5pby9jb21wb25lbnQiOiJweGMiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6InNtYXJ0LXVwZGF0ZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9LCJhbm5vdGF0aW9ucyI6eyJrdWJlY3RsLmt1YmVybmV0ZXMuaW8vZGVmYXVsdC1jb250YWluZXIiOiJweGMiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiNzU4MDYzMWU3MzMwMDA2NjZmNTRiMDFiOThkOTE4ZjciLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6IjIzNTY3MTkwMWNmYmNlYzE0MDgyNmUyZDg4YzI2YmJiIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJ0bXAiLCJlbXB0eURpciI6e319LHsibmFtZSI6ImNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoic21hcnQtdXBkYXRlLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtc3NsLWludGVybmFsIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJzc2wiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6InNtYXJ0LXVwZGF0ZS1zc2wiLCJvcHRpb25hbCI6ZmFsc2V9fSx7Im5hbWUiOiJhdXRvLWNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoiYXV0by1zbWFydC11cGRhdGUtcHhjIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJ2YXVsdC1rZXlyaW5nLXNlY3JldCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoic21hcnQtdXBkYXRlLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXlzcWwtaW5pdC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtbXlzcWwtaW5pdCIsIm9wdGlvbmFsIjp0cnVlfX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjQzNC0zYjY1ZDdmZSIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImRhdGFkaXIiLCJtb3VudFBhdGgiOiIvdmFyL2xpYi9teXNxbCJ9XSwiaW1hZ2VQdWxsUG9saWN5IjoiQWx3YXlzIn1dLCJjb250YWluZXJzIjpbeyJuYW1lIjoicHhjIiwiaW1hZ2UiOiJwZXJjb25hL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXI6OC40LjUtNS4xIiwiY29tbWFuZCI6WyIvdmFyL2xpYi9teXNxbC9weGMtZW50cnlwb2ludC5zaCJdLCJhcmdzIjpbIm15c3FsZCJdLCJwb3J0cyI6W3sibmFtZSI6Im15c3FsIiwiY29udGFpbmVyUG9ydCI6MzMwNn0seyJuYW1lIjoic3N0IiwiY29udGFpbmVyUG9ydCI6NDQ0NH0seyJuYW1lIjoid3JpdGUtc2V0IiwiY29udGFpbmVyUG9ydCI6NDU2N30seyJuYW1lIjoiaXN0IiwiY29udGFpbmVyUG9ydCI6NDU2OH0seyJuYW1lIjoibXlzcWwtYWRtaW4iLCJjb250YWluZXJQb3J0IjozMzA2Mn0seyJuYW1lIjoibXlzcWx4IiwiY29udGFpbmVyUG9ydCI6MzMwNjB9XSwiZW52RnJvbSI6W3sic2VjcmV0UmVmIjp7Im5hbWUiOiJzbWFydC11cGRhdGUtZW52LXZhcnMtcHhjIiwib3B0aW9uYWwiOnRydWV9fV0sImVudiI6W3sibmFtZSI6IlBYQ19TRVJWSUNFIiwidmFsdWUiOiJzbWFydC11cGRhdGUtcHhjLXVucmVhZHkifSx7Im5hbWUiOiJNT05JVE9SX0hPU1QiLCJ2YWx1ZSI6IiUifSx7Im5hbWUiOiJNWVNRTF9ST09UX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5Ijoicm9vdCJ9fX0seyJuYW1lIjoiWFRSQUJBQ0tVUF9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6Inh0cmFiYWNrdXAifX19LHsibmFtZSI6Ik1PTklUT1JfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJtb25pdG9yIn19fSx7Im5hbWUiOiJDTFVTVEVSX0hBU0giLCJ2YWx1ZSI6IjI1ODAwNDAifSx7Im5hbWUiOiJPUEVSQVRPUl9BRE1JTl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6Im9wZXJhdG9yIn19fSx7Im5hbWUiOiJMSVZFTkVTU19DSEVDS19USU1FT1VUIiwidmFsdWUiOiI1In0seyJuYW1lIjoiUkVBRElORVNTX0NIRUNLX1RJTUVPVVQiLCJ2YWx1ZSI6IjE1In0seyJuYW1lIjoiREVGQVVMVF9BVVRIRU5USUNBVElPTl9QTFVHSU4iLCJ2YWx1ZSI6ImNhY2hpbmdfc2hhMl9wYXNzd29yZCJ9LHsibmFtZSI6Ik1ZU1FMX05PVElGWV9TT0NLRVQiLCJ2YWx1ZSI6Ii92YXIvbGliL215c3FsL25vdGlmeS5zb2NrIn0seyJu"..., - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzbWFydC11cGRhdGUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifX0sInRlbXBsYXRlIjp7Im1ldGFkYXRhIjp7ImxhYmVscyI6eyJhcHAua3ViZXJuZXRlcy5pby9jb21wb25lbnQiOiJweGMiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6InNtYXJ0LXVwZGF0ZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9LCJhbm5vdGF0aW9ucyI6eyJrdWJlY3RsLmt1YmVybmV0ZXMuaW8vZGVmYXVsdC1jb250YWluZXIiOiJweGMiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiNzU4MDYzMWU3MzMwMDA2NjZmNTRiMDFiOThkOTE4ZjciLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6IjIzNTY3MTkwMWNmYmNlYzE0MDgyNmUyZDg4YzI2YmJiIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJ0bXAiLCJlbXB0eURpciI6e319LHsibmFtZSI6ImNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoic21hcnQtdXBkYXRlLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtc3NsLWludGVybmFsIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJzc2wiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6InNtYXJ0LXVwZGF0ZS1zc2wiLCJvcHRpb25hbCI6ZmFsc2V9fSx7Im5hbWUiOiJhdXRvLWNvbmZpZyIsImNvbmZpZ01hcCI6eyJuYW1lIjoiYXV0by1zbWFydC11cGRhdGUtcHhjIiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJ2YXVsdC1rZXlyaW5nLXNlY3JldCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoic21hcnQtdXBkYXRlLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXlzcWwtaW5pdC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzbWFydC11cGRhdGUtbXlzcWwtaW5pdCIsIm9wdGlvbmFsIjp0cnVlfX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjQzNC0zYjY1ZDdmZSIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImRhdGFkaXIiLCJtb3VudFBhdGgiOiIvdmFyL2xpYi9teXNxbCJ9XSwiaW1hZ2VQdWxsUG9saWN5IjoiQWx3YXlzIn1dLCJjb250YWluZXJzIjpbeyJuYW1lIjoicHhjIiwiaW1hZ2UiOiJwZXJjb25hL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXI6OC40LjUtNS4xIiwiY29tbWFuZCI6WyIvdmFyL2xpYi9teXNxbC9weGMtZW50cnlwb2ludC5zaCJdLCJhcmdzIjpbIm15c3FsZCJdLCJwb3J0cyI6W3sibmFtZSI6Im15c3FsIiwiY29udGFpbmVyUG9ydCI6MzMwNn0seyJuYW1lIjoic3N0IiwiY29udGFpbmVyUG9ydCI6NDQ0NH0seyJuYW1lIjoid3JpdGUtc2V0IiwiY29udGFpbmVyUG9ydCI6NDU2N30seyJuYW1lIjoiaXN0IiwiY29udGFpbmVyUG9ydCI6NDU2OH0seyJuYW1lIjoibXlzcWwtYWRtaW4iLCJjb250YWluZXJQb3J0IjozMzA2Mn0seyJuYW1lIjoibXlzcWx4IiwiY29udGFpbmVyUG9ydCI6MzMwNjB9XSwiZW52RnJvbSI6W3sic2VjcmV0UmVmIjp7Im5hbWUiOiJzbWFydC11cGRhdGUtZW52LXZhcnMtcHhjIiwib3B0aW9uYWwiOnRydWV9fV0sImVudiI6W3sibmFtZSI6IlBYQ19TRVJWSUNFIiwidmFsdWUiOiJzbWFydC11cGRhdGUtcHhjLXVucmVhZHkifSx7Im5hbWUiOiJNT05JVE9SX0hPU1QiLCJ2YWx1ZSI6IiUifSx7Im5hbWUiOiJNWVNRTF9ST09UX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtc21hcnQtdXBkYXRlIiwia2V5Ijoicm9vdCJ9fX0seyJuYW1lIjoiWFRSQUJBQ0tVUF9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6Inh0cmFiYWNrdXAifX19LHsibmFtZSI6Ik1PTklUT1JfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1zbWFydC11cGRhdGUiLCJrZXkiOiJtb25pdG9yIn19fSx7Im5hbWUiOiJDTFVTVEVSX0hBU0giLCJ2YWx1ZSI6IjQwNjU5MjgifSx7Im5hbWUiOiJPUEVSQVRPUl9BRE1JTl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLXNtYXJ0LXVwZGF0ZSIsImtleSI6Im9wZXJhdG9yIn19fSx7Im5hbWUiOiJMSVZFTkVTU19DSEVDS19USU1FT1VUIiwidmFsdWUiOiI1In0seyJuYW1lIjoiUkVBRElORVNTX0NIRUNLX1RJTUVPVVQiLCJ2YWx1ZSI6IjE1In0seyJuYW1lIjoiREVGQVVMVF9BVVRIRU5USUNBVElPTl9QTFVHSU4iLCJ2YWx1ZSI6ImNhY2hpbmdfc2hhMl9wYXNzd29yZCJ9LHsibmFtZSI6Ik1ZU1FMX05PVElGWV9TT0NLRVQiLCJ2YWx1ZSI6Ii92YXIvbGliL215c3FsL25vdGlmeS5zb2NrIn0seyJu"..., "/percona-xtradb-cluster", + PeriodSeconds: 0, - PeriodSeconds: 10, + PersistentVolumeClaimRetentionPolicy: nil, - PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}", + Phase: "", - Phase: "Pending", + PodManagementPolicy: "", - PodManagementPolicy: "OrderedReady", Ports: []v1.ContainerPort{ PreemptionPolicy: nil, ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}}, + Protocol: "", - Protocol: "TCP", Quobyte: nil, ReadinessProbe: &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...}, + ReadyReplicas: 0, - ReadyReplicas: 3, + Replicas: 0, Replicas: &3, - Replicas: 3, + ResourceVersion: "", - ResourceVersion: "1775708517867039017", - ResourceVersion: "1775709163985135017", - ResourceVersion: "1775709755503327017", + RestartPolicy: "", - RestartPolicy: "Always", - RevisionHistoryLimit: &10, + RevisionHistoryLimit: nil, + SchedulerName: "", - SchedulerName: "default-scheduler", SecretName: "internal-smart-update", SecretName: "smart-update-mysql-init", SecretName: "smart-update-ssl", SecretName: "smart-update-ssl-internal", SecretName: "smart-update-vault", Secret: &v1.SecretVolumeSource{ SecurityContext: nil, Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "smart-update", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, SelfLink: "", ServiceAccountName: "default", ServiceName: "smart-update-pxc", SetHostnameAsFQDN: nil, sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 Spec: v1.PersistentVolumeClaimSpec{ Spec: v1.PodSpec{ Spec: v1.StatefulSetSpec{ StartupProbe: nil, Status: v1.PersistentVolumeClaimStatus{ Status: v1.StatefulSetStatus{ StorageClassName: nil, Subdomain: "", - Subresource: "status", SuccessThreshold: 1, Template: v1.PodTemplateSpec{ TerminationGracePeriodSeconds: &600, TerminationGracePeriodSeconds: nil, + TerminationMessagePath: "", - TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "", - TerminationMessagePolicy: "File", TimeoutSeconds: 5, - Time: s"2026-04-09 04:17:55 +0000 UTC", - Time: s"2026-04-09 04:21:57 +0000 UTC", - Time: s"2026-04-09 04:28:48 +0000 UTC", - Time: s"2026-04-09 04:32:43 +0000 UTC", - Time: s"2026-04-09 04:38:37 +0000 UTC", - Time: s"2026-04-09 04:42:35 +0000 UTC", Tolerations: nil, - TopologySpreadConstraints: nil, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, TypeMeta: {}, TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"}, + UID: "", - UID: "a19b49e7-16a2-4164-82a7-db451343461e", - UID: "adadc58d-b3bb-4260-ada8-8c0ea8215f70", - UID: "c89f78f7-c55c-43d5-aa0b-a310a636bd99", + UpdatedReplicas: 0, - UpdatedReplicas: 3, + UpdateRevision: "", - UpdateRevision: "smart-update-pxc-5778b47548", - UpdateRevision: "smart-update-pxc-5d5b84b669", - UpdateRevision: "smart-update-pxc-86bdb8fd67", UpdateStrategy: {Type: "OnDelete"}, &v1.StatefulSet{ VolumeAttributesClassName: nil, VolumeClaimTemplates: []v1.PersistentVolumeClaim{ VolumeDevices: nil, - VolumeMode: &"Filesystem", + VolumeMode: nil, VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...}, VolumeName: "", VolumeSource: v1.VolumeSource{ Volumes: []v1.Volume{ VsphereVolume: nil, WorkingDir: "", + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.jT4fgXG2Cs ++ mktemp + local LAST_ERR=/tmp/tmp.NjwWE7KqxZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jT4fgXG2Cs No resources found + cat /tmp/tmp.NjwWE7KqxZ + rm /tmp/tmp.jT4fgXG2Cs /tmp/tmp.NjwWE7KqxZ + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.5AM7z2V0UO ++ mktemp + local LAST_ERR=/tmp/tmp.JYfNMG27on + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5AM7z2V0UO No resources found + cat /tmp/tmp.JYfNMG27on + rm /tmp/tmp.5AM7z2V0UO /tmp/tmp.JYfNMG27on + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.yyX13u5mOH ++ mktemp + local LAST_ERR=/tmp/tmp.KDsHLRn38L + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yyX13u5mOH No resources found + cat /tmp/tmp.KDsHLRn38L + rm /tmp/tmp.yyX13u5mOH /tmp/tmp.KDsHLRn38L + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.n6IgCbStnb ++ mktemp + local LAST_ERR=/tmp/tmp.RabLwRcqdB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.n6IgCbStnb validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.RabLwRcqdB + rm /tmp/tmp.n6IgCbStnb /tmp/tmp.RabLwRcqdB + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace smart-update1-11246 + rm -rf /tmp/tmp.Kmewg0QhXK + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.YyzS6Bj5I7 + desc 'test passed' + set +o xtrace + local LAST_OUT=/tmp/tmp.IvxOyjka02 ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.WKK6HNdCFG + local exit_status=0 + local LAST_ERR=/tmp/tmp.1nEwdVfc4h ++ seq 0 2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace smart-update1-11246 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator