Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/logs/smart-update2-8-0.log Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 + API=pxc.percona.com/v9-9-9 + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + CLUSTER=smart-update + CLUSTER_SIZE=3 + PROXY_SIZE=2 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + PXC_VER=8.0 + TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + VS_URL=http://version-service + VS_PORT=11000 + VS_ENDPOINT=http://version-service:11000 + [[ 8.0 == \8\.\4 ]] + VS_UPDATE_STRATEGY=recommended + main + create_infra smart-update2-13972 + local ns=smart-update2-13972 + '[' -n pxc-operator ']' + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + [[ -z smart-update2-9397 ]] + [[ -z minimal-cluster ]] + kubectl_bin patch pxc -n smart-update2-9397 minimal-cluster --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.kxRxnZwXKo ++ mktemp + local LAST_ERR=/tmp/tmp.6p7CalGMZP + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc -n smart-update2-9397 minimal-cluster --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kxRxnZwXKo perconaxtradbcluster.pxc.percona.com/minimal-cluster patched (no change) + cat /tmp/tmp.6p7CalGMZP + rm /tmp/tmp.kxRxnZwXKo /tmp/tmp.6p7CalGMZP + return 0 + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-9397 namespace + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ awk '-F ' '{print $2}' ++ tail -n1 ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.c0GN1zCCLJ + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + xargs kubectl delete ns + local LAST_ERR=/tmp/tmp.wzoZ8jGu0g + local exit_status=0 + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.xlTjR2PS5Z ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace pxc-operator ++ mktemp + local LAST_ERR=/tmp/tmp.TodxXmbni9 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xlTjR2PS5Z + cat /tmp/tmp.TodxXmbni9 + rm /tmp/tmp.xlTjR2PS5Z /tmp/tmp.TodxXmbni9 + return 0 namespace "cert-manager" deleted namespace "smart-update2-9397" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.c0GN1zCCLJ namespace "pxc-operator" deleted + cat /tmp/tmp.wzoZ8jGu0g + rm /tmp/tmp.c0GN1zCCLJ /tmp/tmp.wzoZ8jGu0g + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.23ZAxQQXiu ++ mktemp + local LAST_ERR=/tmp/tmp.ZiTlTRv8Ty + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.23ZAxQQXiu namespace/pxc-operator created + cat /tmp/tmp.ZiTlTRv8Ty + rm /tmp/tmp.23ZAxQQXiu /tmp/tmp.ZiTlTRv8Ty + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.tn5VeeRlOo +++ mktemp ++ local LAST_ERR=/tmp/tmp.KBMwquwFny ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tn5VeeRlOo ++ cat /tmp/tmp.KBMwquwFny ++ rm /tmp/tmp.tn5VeeRlOo /tmp/tmp.KBMwquwFny ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster8 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.4H30eLmY11 ++ mktemp + local LAST_ERR=/tmp/tmp.jkg1oPOKQX + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster8 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4H30eLmY11 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster8" modified. + cat /tmp/tmp.jkg1oPOKQX + rm /tmp/tmp.4H30eLmY11 /tmp/tmp.jkg1oPOKQX + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.nj9iqfSsse ++ mktemp + local LAST_ERR=/tmp/tmp.XrH8ZRN0g1 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nj9iqfSsse customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.XrH8ZRN0g1 + rm /tmp/tmp.nj9iqfSsse /tmp/tmp.XrH8ZRN0g1 + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.X1fa8jqT34 ++ mktemp + local LAST_ERR=/tmp/tmp.Ldu3CAbBi6 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.X1fa8jqT34 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.Ldu3CAbBi6 + rm /tmp/tmp.X1fa8jqT34 /tmp/tmp.Ldu3CAbBi6 + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.k7egrzwhIU ++ mktemp + local LAST_ERR=/tmp/tmp.taRPdwNKXQ + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.k7egrzwhIU customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.taRPdwNKXQ + rm /tmp/tmp.k7egrzwhIU /tmp/tmp.taRPdwNKXQ + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.TCe7UyopWO ++ mktemp + local LAST_ERR=/tmp/tmp.zmBx2lUCvw + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TCe7UyopWO customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.zmBx2lUCvw + rm /tmp/tmp.TCe7UyopWO /tmp/tmp.zmBx2lUCvw + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.VMJuXx3qiZ ++ mktemp + local LAST_ERR=/tmp/tmp.ixySu4xb4l + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VMJuXx3qiZ clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.ixySu4xb4l + rm /tmp/tmp.VMJuXx3qiZ /tmp/tmp.ixySu4xb4l + return 0 + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' ++ mktemp + local LAST_OUT=/tmp/tmp.4c14EX2hHq + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420^' ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-operator.yaml + local LAST_ERR=/tmp/tmp.ZtnPtkMV64 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4c14EX2hHq deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.ZtnPtkMV64 + rm /tmp/tmp.4c14EX2hHq /tmp/tmp.ZtnPtkMV64 + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.Lo3ODqiqFJ ++ mktemp + local LAST_ERR=/tmp/tmp.R6g8TV24bb + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Lo3ODqiqFJ pod/percona-xtradb-cluster-operator-658fcf4b98-5vms7 condition met + cat /tmp/tmp.R6g8TV24bb + rm /tmp/tmp.Lo3ODqiqFJ /tmp/tmp.R6g8TV24bb + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ local LAST_OUT=/tmp/tmp.M7lfWqAqt4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.szdS5yWTf2 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.M7lfWqAqt4 ++ cat /tmp/tmp.szdS5yWTf2 ++ rm /tmp/tmp.M7lfWqAqt4 /tmp/tmp.szdS5yWTf2 ++ return 0 + wait_pod percona-xtradb-cluster-operator-658fcf4b98-5vms7 480 pxc-operator + local pod=percona-xtradb-cluster-operator-658fcf4b98-5vms7 + local max_retry=480 + local ns=pxc-operator ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo percona-xtradb-cluster-operator-658fcf4b98-5vms7 + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-658fcf4b98-5vms7 condition met waiting for pod/percona-xtradb-cluster-operator-658fcf4b98-5vms7 to become Ready.Ok + sleep 3 + create_namespace smart-update2-13972 + local namespace=smart-update2-13972 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl api-resources ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrole ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces smart-update2-13972' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces smart-update2-13972 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace smart-update2-13972 ++ mktemp + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.WZ7QZHBuMT + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.g7uDvxqNuw + local exit_status=0 + local LAST_OUT=/tmp/tmp.IEGnqaMZKK ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace smart-update2-13972 ++ mktemp + local LAST_ERR=/tmp/tmp.MRZbid0f6o + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace smart-update2-13972 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IEGnqaMZKK + cat /tmp/tmp.MRZbid0f6o + rm /tmp/tmp.IEGnqaMZKK /tmp/tmp.MRZbid0f6o + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace smart-update2-13972 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.WZ7QZHBuMT + cat /tmp/tmp.g7uDvxqNuw Error from server (NotFound): namespaces "smart-update2-13972" not found + rm /tmp/tmp.WZ7QZHBuMT /tmp/tmp.g7uDvxqNuw + return 1 + : + wait_for_delete namespace/smart-update2-13972 + local res=namespace/smart-update2-13972 + echo -n 'waiting for namespace/smart-update2-13972 to be deleted' waiting for namespace/smart-update2-13972 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "smart-update2-13972" not found + desc 'create namespace smart-update2-13972' + set +o xtrace ----------------------------------------------------------------------------------- create namespace smart-update2-13972 ----------------------------------------------------------------------------------- + kubectl_bin create namespace smart-update2-13972 ++ mktemp + local LAST_OUT=/tmp/tmp.xeTrsFWEwY ++ mktemp + local LAST_ERR=/tmp/tmp.ep7lYw6cIJ + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace smart-update2-13972 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xeTrsFWEwY namespace/smart-update2-13972 created + cat /tmp/tmp.ep7lYw6cIJ + rm /tmp/tmp.xeTrsFWEwY /tmp/tmp.ep7lYw6cIJ + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.CKsEnm1CV9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.uzENfSqytj ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CKsEnm1CV9 ++ cat /tmp/tmp.uzENfSqytj ++ rm /tmp/tmp.CKsEnm1CV9 /tmp/tmp.uzENfSqytj ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster8 --namespace=smart-update2-13972 ++ mktemp + local LAST_OUT=/tmp/tmp.WkpbNwqWNB ++ mktemp + local LAST_ERR=/tmp/tmp.SmO4mVXb2H + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster8 --namespace=smart-update2-13972 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WkpbNwqWNB Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster8" modified. + cat /tmp/tmp.SmO4mVXb2H + rm /tmp/tmp.WkpbNwqWNB /tmp/tmp.SmO4mVXb2H + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xoe8PQ9dmA ++ mktemp + local LAST_ERR=/tmp/tmp.y4ySI0Etve + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xoe8PQ9dmA secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.y4ySI0Etve + rm /tmp/tmp.xoe8PQ9dmA /tmp/tmp.y4ySI0Etve + return 0 + deploy_version_service + desc 'install version service' + set +o xtrace ----------------------------------------------------------------------------------- install version service ----------------------------------------------------------------------------------- + kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json ++ mktemp + local LAST_OUT=/tmp/tmp.ocR6O9X0Js ++ mktemp + local LAST_ERR=/tmp/tmp.aXc4Bw4T67 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ocR6O9X0Js configmap/versions created + cat /tmp/tmp.aXc4Bw4T67 + rm /tmp/tmp.ocR6O9X0Js /tmp/tmp.aXc4Bw4T67 + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.1vhyNM6g69 ++ mktemp + local LAST_ERR=/tmp/tmp.MoZeIV8VzZ + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1vhyNM6g69 deployment.apps/version-service created service/version-service created + cat /tmp/tmp.MoZeIV8VzZ + rm /tmp/tmp.1vhyNM6g69 /tmp/tmp.MoZeIV8VzZ + return 0 + sleep 10 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.yqrVfdlIyi ++ mktemp + local LAST_ERR=/tmp/tmp.HrCfAf83vR + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yqrVfdlIyi namespace/cert-manager created + cat /tmp/tmp.HrCfAf83vR + rm /tmp/tmp.yqrVfdlIyi /tmp/tmp.HrCfAf83vR + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.em8YuFr5HX ++ mktemp + local LAST_ERR=/tmp/tmp.Nq7EMbEMW2 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.em8YuFr5HX namespace/cert-manager labeled + cat /tmp/tmp.Nq7EMbEMW2 + rm /tmp/tmp.em8YuFr5HX /tmp/tmp.Nq7EMbEMW2 + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.wXbqd1EBXk ++ mktemp + local LAST_ERR=/tmp/tmp.LbgdscetAw + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wXbqd1EBXk namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.LbgdscetAw Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.wXbqd1EBXk /tmp/tmp.LbgdscetAw + return 0 + '[' '' == 4.10 ']' + kubectl_bin -n cert-manager rollout status deployment/cert-manager --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.5Ai0QGZx3x ++ mktemp + local LAST_ERR=/tmp/tmp.RFlJ3TfG7Q + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager rollout status deployment/cert-manager --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5Ai0QGZx3x deployment "cert-manager" successfully rolled out + cat /tmp/tmp.RFlJ3TfG7Q + rm /tmp/tmp.5Ai0QGZx3x /tmp/tmp.RFlJ3TfG7Q + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.JB8ILSrA9A ++ mktemp + local LAST_ERR=/tmp/tmp.m7Aa26IJDj + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JB8ILSrA9A deployment "cert-manager-cainjector" successfully rolled out + cat /tmp/tmp.m7Aa26IJDj + rm /tmp/tmp.JB8ILSrA9A /tmp/tmp.m7Aa26IJDj + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.dm86emXisJ ++ mktemp + local LAST_ERR=/tmp/tmp.0wjEMVi0zS + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dm86emXisJ deployment "cert-manager-webhook" successfully rolled out + cat /tmp/tmp.0wjEMVi0zS + rm /tmp/tmp.dm86emXisJ /tmp/tmp.0wjEMVi0zS + return 0 + log 'waiting for cert-manager webhook caBundle to be injected' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:30:54+0000]' waiting for cert-manager webhook caBundle to be injected [2026-10-05T14:30:54+0000] waiting for cert-manager webhook caBundle to be injected ++ date +%s + local deadline=1791210774 + kubectl_bin get validatingwebhookconfigurations cert-manager-webhook -o 'jsonpath={.webhooks[0].clientConfig.caBundle}' + grep -q . + log 'probing cert-manager webhook' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:30:55+0000]' probing cert-manager webhook [2026-10-05T14:30:55+0000] probing cert-manager webhook + local probe_file ++ mktemp + probe_file=/tmp/tmp.HMrJ3hXTIx + cat + local retries=0 + kubectl_bin apply -f /tmp/tmp.HMrJ3hXTIx --dry-run=server + rm -f /tmp/tmp.HMrJ3hXTIx ++ grep :8.0 ++ tail -n3 ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ sort -V +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++ head -n1 ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++++ grep -c percona-xtradb-cluster-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ head -1 ++++ mktemp +++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ local LAST_OUT=/tmp/tmp.voiuN7gK2f ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oQU5583eca +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.voiuN7gK2f +++ cat /tmp/tmp.oQU5583eca +++ rm /tmp/tmp.voiuN7gK2f /tmp/tmp.oQU5583eca +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-658fcf4b98-5vms7 -n pxc-operator -- curl -s http://version-service.smart-update2-13972.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.GchhXKaGma +++ mktemp ++ local LAST_ERR=/tmp/tmp.nj5F1Q6JrG ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-658fcf4b98-5vms7 -n pxc-operator -- curl -s http://version-service.smart-update2-13972.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GchhXKaGma ++ cat /tmp/tmp.nj5F1Q6JrG Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.GchhXKaGma /tmp/tmp.nj5F1Q6JrG ++ return 0 + IMAGE_PXC=percona/percona-xtradb-cluster:8.0.41-32.1 + kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' ++ mktemp + local LAST_OUT=/tmp/tmp.VqiRsFKMzB ++ mktemp + local LAST_ERR=/tmp/tmp.UbLIhWB0s1 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VqiRsFKMzB customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched + cat /tmp/tmp.UbLIhWB0s1 + rm /tmp/tmp.VqiRsFKMzB /tmp/tmp.UbLIhWB0s1 + return 0 + kubectl_bin -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-13972.svc.cluster.local:11000 ++ mktemp + local LAST_OUT=/tmp/tmp.SIGm1ZVMTr ++ mktemp + local LAST_ERR=/tmp/tmp.H3kcnHc5Pg + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-13972.svc.cluster.local:11000 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SIGm1ZVMTr deployment.apps/percona-xtradb-cluster-operator env updated + cat /tmp/tmp.H3kcnHc5Pg + rm /tmp/tmp.SIGm1ZVMTr /tmp/tmp.H3kcnHc5Pg + return 0 + desc 'Starting telemetry testing' + set +o xtrace ----------------------------------------------------------------------------------- Starting telemetry testing ----------------------------------------------------------------------------------- + /usr/bin/sed s/version-service/version-service-cr/g /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/vs.yml + kubectl_bin apply -f - ++ mktemp ++ yq 'select(.kind == "Deployment").spec.template.spec.containers[0].image' /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/vs.yml + local LAST_OUT=/tmp/tmp.PidDj3GmrI ++ mktemp + local LAST_ERR=/tmp/tmp.1avsG6t8XY + local exit_status=0 + yq eval '(. | select(.kind == "Deployment") | .spec.template.spec.containers[0].image) = "perconalab/version-service:main-latest"' ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PidDj3GmrI deployment.apps/version-service-cr created service/version-service-cr created + cat /tmp/tmp.1avsG6t8XY + rm /tmp/tmp.PidDj3GmrI /tmp/tmp.1avsG6t8XY + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.CuSklh6c9w ++ mktemp + local LAST_ERR=/tmp/tmp.KRFp2EfVJZ + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CuSklh6c9w pod "version-service-66774c865b-jqwss" deleted from smart-update2-13972 namespace + cat /tmp/tmp.KRFp2EfVJZ + rm /tmp/tmp.CuSklh6c9w /tmp/tmp.KRFp2EfVJZ + return 0 ++ sed -r 's/^.*:([0-9]+.[0-9]+).*/\1/' ++ echo -n percona/percona-xtradb-cluster:8.0.41-32.1 + IMAGE_PREFIX=8.0 + desc 'Enable telemetry on operator level' + set +o xtrace ----------------------------------------------------------------------------------- Enable telemetry on operator level ----------------------------------------------------------------------------------- + kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "false"' + kubectl_bin apply -n pxc-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.UAzAT5H5Bf ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.IPawib9Drt ++ mktemp + local LAST_ERR=/tmp/tmp.oe9dwiv00u + local exit_status=0 + local LAST_ERR=/tmp/tmp.TZPyZ1iCHx + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n pxc-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UAzAT5H5Bf + cat /tmp/tmp.oe9dwiv00u + rm /tmp/tmp.UAzAT5H5Bf /tmp/tmp.oe9dwiv00u + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IPawib9Drt deployment.apps/percona-xtradb-cluster-operator configured + cat /tmp/tmp.TZPyZ1iCHx + rm /tmp/tmp.IPawib9Drt /tmp/tmp.TZPyZ1iCHx + return 0 + sleep 30 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ local LAST_OUT=/tmp/tmp.MD63vnMVSJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.uFjSomWXvt ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MD63vnMVSJ ++ cat /tmp/tmp.uFjSomWXvt ++ rm /tmp/tmp.MD63vnMVSJ /tmp/tmp.uFjSomWXvt ++ return 0 + wait_pod percona-xtradb-cluster-operator-678f797689-gtw2h 480 pxc-operator + local pod=percona-xtradb-cluster-operator-678f797689-gtw2h + local max_retry=480 + local ns=pxc-operator ++ grep -E '^(pxc|proxysql)$' ++ echo percona-xtradb-cluster-operator-678f797689-gtw2h ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-678f797689-gtw2h condition met waiting for pod/percona-xtradb-cluster-operator-678f797689-gtw2h to become Ready.Ok + check_telemetry_transfer http://version-service-cr.smart-update2-13972.svc.cluster.local:11000 disabled enabled + local cr_vs_uri=http://version-service-cr.smart-update2-13972.svc.cluster.local:11000 + local cr_vs_channel=disabled + local telemetry_state=enabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.8h2OVYIOhT ++ mktemp + local LAST_ERR=/tmp/tmp.FyWCfZqLbD + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8h2OVYIOhT deployment.apps/pxc-client created + cat /tmp/tmp.FyWCfZqLbD + rm /tmp/tmp.8h2OVYIOhT /tmp/tmp.FyWCfZqLbD + return 0 + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.RhhAPuTzvL ++ mktemp + local LAST_ERR=/tmp/tmp.qZMxBasbkp + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RhhAPuTzvL secret/minimal-cluster created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.qZMxBasbkp + rm /tmp/tmp.RhhAPuTzvL /tmp/tmp.qZMxBasbkp + return 0 + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-13972.svc.cluster.local:11000"' + kubectl_bin apply -f - + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.41-32.1"' + yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"' + yq eval '.spec.crVersion="9.9.9"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cr-minimal.yaml + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420"' + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + yq eval '.spec.upgradeOptions.apply="disabled"' ++ mktemp + local LAST_OUT=/tmp/tmp.qpESdfTkao ++ mktemp + local LAST_ERR=/tmp/tmp.x4rBtXBdK8 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qpESdfTkao perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.x4rBtXBdK8 + rm /tmp/tmp.qpESdfTkao /tmp/tmp.x4rBtXBdK8 + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-pxc ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-pxc ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in $(seq 0 $last_pod) + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.d5PVCHaTi7 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FhoBvBalnC +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.d5PVCHaTi7 +++ cat /tmp/tmp.FhoBvBalnC +++ rm /tmp/tmp.d5PVCHaTi7 /tmp/tmp.FhoBvBalnC +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-haproxy ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-haproxy ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in $(seq 0 $last_pod) + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-haproxy-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.A4xIXMYeEV +++ mktemp ++ local LAST_ERR=/tmp/tmp.PXhbhYqxjt ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A4xIXMYeEV ++ cat /tmp/tmp.PXhbhYqxjt ++ rm /tmp/tmp.A4xIXMYeEV /tmp/tmp.PXhbhYqxjt ++ return 0 + client_pod=pxc-client-6fc75f888d-w92tw + wait_pod pxc-client-6fc75f888d-w92tw + local pod=pxc-client-6fc75f888d-w92tw + local max_retry=480 + local ns= ++ echo pxc-client-6fc75f888d-w92tw ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-w92tw condition met waiting for pod/pxc-client-6fc75f888d-w92tw to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.G5j9WIa4f8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.0mPCaoZqpJ ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.G5j9WIa4f8 ++ cat /tmp/tmp.0mPCaoZqpJ ++ rm /tmp/tmp.G5j9WIa4f8 /tmp/tmp.0mPCaoZqpJ ++ return 0 + client_pod=pxc-client-6fc75f888d-w92tw + wait_pod pxc-client-6fc75f888d-w92tw + local pod=pxc-client-6fc75f888d-w92tw + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6fc75f888d-w92tw ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-w92tw condition met waiting for pod/pxc-client-6fc75f888d-w92tw to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.time_ms")' + grep -Eo '\{.*\}' + grep -E 'server request payload|unary call' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-cr-58b8ffbbb4-q58wp ++ mktemp + local LAST_OUT=/tmp/tmp.1gVJvU2RJj ++ mktemp + local LAST_ERR=/tmp/tmp.pB4ZOni5mW + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs version-service-cr-58b8ffbbb4-q58wp + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1gVJvU2RJj + cat /tmp/tmp.pB4ZOni5mW + rm /tmp/tmp.1gVJvU2RJj /tmp/tmp.pB4ZOni5mW + return 0 + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.start_time")' + grep -Eo '\{.*\}' + jq 'del(."grpc.request.content".msg.kubeVersion)' + grep -E 'server request payload|unary call' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-66774c865b-wxh67 ++ mktemp + local LAST_OUT=/tmp/tmp.p2LT7PWhHq ++ mktemp + local LAST_ERR=/tmp/tmp.CcU2Ikkl2W + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs version-service-66774c865b-wxh67 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.p2LT7PWhHq + cat /tmp/tmp.CcU2Ikkl2W + rm /tmp/tmp.p2LT7PWhHq /tmp/tmp.CcU2Ikkl2W + return 0 + local telemetry_log_file=enabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' disabled == disabled -a enabled == enabled ']' + desc 'operator fallback VS should have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator fallback VS should have telemetry ----------------------------------------------------------------------------------- + diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json /dev/fd/63 ++ grep -f /tmp/tmp.WkA2TS2tBT/enabled_telemetry.version-service.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json + desc 'CR VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- CR VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.WkA2TS2tBT/enabled_telemetry.version-service-cr.log.json ]] + local image_prefix=disabled + image_prefix=disabled + local telemetry_cr_log_file=enabled_telemetry.version-service-cr-disabled-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + [[ disabled == \d\i\s\a\b\l\e\d\-\r\e\c\o\m\m\e\n\d\e\d ]] + [[ disabled == \d\i\s\a\b\l\e\d\-\l\a\t\e\s\t ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' disabled == disabled -a enabled == disabled ']' + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.x9hEOPEUa0 ++ mktemp + local LAST_ERR=/tmp/tmp.bUKfn2BgsO + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.x9hEOPEUa0 perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.bUKfn2BgsO + rm /tmp/tmp.x9hEOPEUa0 /tmp/tmp.bUKfn2BgsO + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.B1yKvoiTSa +++ mktemp ++ local LAST_ERR=/tmp/tmp.qho02Nspgm ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.B1yKvoiTSa ++ cat /tmp/tmp.qho02Nspgm ++ rm /tmp/tmp.B1yKvoiTSa /tmp/tmp.qho02Nspgm ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-678f797689-gtw2h ++ mktemp + local LAST_OUT=/tmp/tmp.kpE2uXVGMK ++ mktemp + local LAST_ERR=/tmp/tmp.BOlqJhlUS3 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-678f797689-gtw2h + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kpE2uXVGMK pod "percona-xtradb-cluster-operator-678f797689-gtw2h" deleted from pxc-operator namespace + cat /tmp/tmp.BOlqJhlUS3 + rm /tmp/tmp.kpE2uXVGMK /tmp/tmp.BOlqJhlUS3 + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.w4Hg1zAsId ++ mktemp + local LAST_ERR=/tmp/tmp.VhwfDn65RL + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.w4Hg1zAsId perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-13972 namespace + cat /tmp/tmp.VhwfDn65RL + rm /tmp/tmp.w4Hg1zAsId /tmp/tmp.VhwfDn65RL + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.eIas2czIF0 ++ mktemp + local LAST_ERR=/tmp/tmp.XLf7iujk6I + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eIas2czIF0 deployment.apps "pxc-client" deleted from smart-update2-13972 namespace + cat /tmp/tmp.XLf7iujk6I + rm /tmp/tmp.eIas2czIF0 /tmp/tmp.XLf7iujk6I + return 0 + sleep 30 + desc 'Disabling telemetry on the operator level' + set +o xtrace ----------------------------------------------------------------------------------- Disabling telemetry on the operator level ----------------------------------------------------------------------------------- + kubectl_bin delete pod -l run=version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.bjjXAF3fpf ++ mktemp + local LAST_ERR=/tmp/tmp.vePCpwvn5V + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pod -l run=version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bjjXAF3fpf pod "version-service-cr-58b8ffbbb4-q58wp" deleted from smart-update2-13972 namespace + cat /tmp/tmp.vePCpwvn5V + rm /tmp/tmp.bjjXAF3fpf /tmp/tmp.vePCpwvn5V + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.GpNpEY2yGT ++ mktemp + local LAST_ERR=/tmp/tmp.P5tyQM31Lo + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GpNpEY2yGT pod "version-service-66774c865b-wxh67" deleted from smart-update2-13972 namespace + cat /tmp/tmp.P5tyQM31Lo + rm /tmp/tmp.GpNpEY2yGT /tmp/tmp.P5tyQM31Lo + return 0 + kubectl_bin apply -n pxc-operator -f - + yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' + kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.0TJvBU9Sxf ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.u4ffPFjv0R + local LAST_ERR=/tmp/tmp.bw5KFe4vk5 + local exit_status=0 ++ mktemp + local LAST_ERR=/tmp/tmp.eMIEJSOJyh + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n pxc-operator -f - ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.u4ffPFjv0R + cat /tmp/tmp.eMIEJSOJyh + rm /tmp/tmp.u4ffPFjv0R /tmp/tmp.eMIEJSOJyh + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0TJvBU9Sxf deployment.apps/percona-xtradb-cluster-operator configured + cat /tmp/tmp.bw5KFe4vk5 + rm /tmp/tmp.0TJvBU9Sxf /tmp/tmp.bw5KFe4vk5 + return 0 + sleep 30 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.vU1MUa3yiL +++ mktemp ++ local LAST_ERR=/tmp/tmp.yyInQZHcDH ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vU1MUa3yiL ++ cat /tmp/tmp.yyInQZHcDH ++ rm /tmp/tmp.vU1MUa3yiL /tmp/tmp.yyInQZHcDH ++ return 0 + wait_pod percona-xtradb-cluster-operator-564f9bc786-mpbk6 480 pxc-operator + local pod=percona-xtradb-cluster-operator-564f9bc786-mpbk6 + local max_retry=480 + local ns=pxc-operator ++ grep -E '^(pxc|proxysql)$' ++ echo percona-xtradb-cluster-operator-564f9bc786-mpbk6 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-564f9bc786-mpbk6 condition met waiting for pod/percona-xtradb-cluster-operator-564f9bc786-mpbk6 to become Ready.Ok + check_telemetry_transfer http://version-service-cr.smart-update2-13972.svc.cluster.local:11000 8.0-recommended disabled + local cr_vs_uri=http://version-service-cr.smart-update2-13972.svc.cluster.local:11000 + local cr_vs_channel=8.0-recommended + local telemetry_state=disabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.bipVbm0ZvU ++ mktemp + local LAST_ERR=/tmp/tmp.zRgh5gy7Xd + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bipVbm0ZvU deployment.apps/pxc-client created + cat /tmp/tmp.zRgh5gy7Xd + rm /tmp/tmp.bipVbm0ZvU /tmp/tmp.zRgh5gy7Xd + return 0 + kubectl_bin apply -f - + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' ++ mktemp + local LAST_OUT=/tmp/tmp.L8F8yhFKqh ++ mktemp + local LAST_ERR=/tmp/tmp.GyaJbSJp2s + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.L8F8yhFKqh secret/minimal-cluster unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.GyaJbSJp2s + rm /tmp/tmp.L8F8yhFKqh /tmp/tmp.GyaJbSJp2s + return 0 + yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"' + kubectl_bin apply -f - + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.41-32.1"' + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-13972.svc.cluster.local:11000"' + yq eval '.spec.upgradeOptions.apply="8.0-recommended"' + yq eval '.spec.crVersion="9.9.9"' ++ mktemp + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + local LAST_OUT=/tmp/tmp.UwHvAgFg0D + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cr-minimal.yaml ++ mktemp + local LAST_ERR=/tmp/tmp.ktvWpSWfaP + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UwHvAgFg0D perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.ktvWpSWfaP + rm /tmp/tmp.UwHvAgFg0D /tmp/tmp.ktvWpSWfaP + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-pxc ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-pxc ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in $(seq 0 $last_pod) + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo minimal-cluster-pxc-0 + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JwU49VHDp4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3L7fYs5WBp +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.JwU49VHDp4 +++ cat /tmp/tmp.3L7fYs5WBp +++ rm /tmp/tmp.JwU49VHDp4 /tmp/tmp.3L7fYs5WBp +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-haproxy ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-haproxy ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in $(seq 0 $last_pod) + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo minimal-cluster-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pcrsF9kIde +++ mktemp ++ local LAST_ERR=/tmp/tmp.Udun0ABPMG ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pcrsF9kIde ++ cat /tmp/tmp.Udun0ABPMG ++ rm /tmp/tmp.pcrsF9kIde /tmp/tmp.Udun0ABPMG ++ return 0 + client_pod=pxc-client-6fc75f888d-pkt9t + wait_pod pxc-client-6fc75f888d-pkt9t + local pod=pxc-client-6fc75f888d-pkt9t + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6fc75f888d-pkt9t ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-pkt9t condition met waiting for pod/pxc-client-6fc75f888d-pkt9t to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9wfEeeG7zZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.CHbxRyqcMO ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9wfEeeG7zZ ++ cat /tmp/tmp.CHbxRyqcMO ++ rm /tmp/tmp.9wfEeeG7zZ /tmp/tmp.CHbxRyqcMO ++ return 0 + client_pod=pxc-client-6fc75f888d-pkt9t + wait_pod pxc-client-6fc75f888d-pkt9t + local pod=pxc-client-6fc75f888d-pkt9t + local max_retry=480 + local ns= ++ echo pxc-client-6fc75f888d-pkt9t ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-pkt9t condition met waiting for pod/pxc-client-6fc75f888d-pkt9t to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + grep -E 'server request payload|unary call' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.request.content".msg.kubeVersion)' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + grep -Eo '\{.*\}' + kubectl_bin logs version-service-cr-58b8ffbbb4-s6nkp ++ mktemp + local LAST_OUT=/tmp/tmp.qXAXYNEnnI ++ mktemp + local LAST_ERR=/tmp/tmp.3FR7PGScZR + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs version-service-cr-58b8ffbbb4-s6nkp + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qXAXYNEnnI + cat /tmp/tmp.3FR7PGScZR + rm /tmp/tmp.qXAXYNEnnI /tmp/tmp.3FR7PGScZR + return 0 + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + grep -Eo '\{.*\}' + grep -E 'server request payload|unary call' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-66774c865b-2mtdw ++ mktemp + local LAST_OUT=/tmp/tmp.dEjVBjsthG ++ mktemp + local LAST_ERR=/tmp/tmp.rkMUIIZfS2 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs version-service-66774c865b-2mtdw + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dEjVBjsthG + cat /tmp/tmp.rkMUIIZfS2 + rm /tmp/tmp.dEjVBjsthG /tmp/tmp.rkMUIIZfS2 + return 0 + local telemetry_log_file=disabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' 8.0-recommended == disabled -a disabled == enabled ']' + local image_prefix=8.0 + image_prefix=8.0 + local telemetry_cr_log_file=disabled_telemetry.version-service-cr-8.0-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + [[ 8.0-recommended == \8\.\0\-\r\e\c\o\m\m\e\n\d\e\d ]] + '[' disabled == disabled ']' + desc 'cr VS should have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- cr VS should have telemetry ----------------------------------------------------------------------------------- + diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.0-cw.log.json /dev/fd/63 ++ grep -f /tmp/tmp.WkA2TS2tBT/disabled_telemetry.version-service-cr.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.0-cw.log.json + desc 'operator VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.WkA2TS2tBT/disabled_telemetry.version-service.log.json ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' 8.0-recommended == disabled -a disabled == disabled ']' + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.sPQqIcTZQX ++ mktemp + local LAST_ERR=/tmp/tmp.Tm5quu7fdc + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sPQqIcTZQX perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.Tm5quu7fdc + rm /tmp/tmp.sPQqIcTZQX /tmp/tmp.Tm5quu7fdc + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.4nShAArCIv +++ mktemp ++ local LAST_ERR=/tmp/tmp.g0PbGkVTCM ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4nShAArCIv ++ cat /tmp/tmp.g0PbGkVTCM ++ rm /tmp/tmp.4nShAArCIv /tmp/tmp.g0PbGkVTCM ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-564f9bc786-mpbk6 ++ mktemp + local LAST_OUT=/tmp/tmp.qHbk2ae6eN ++ mktemp + local LAST_ERR=/tmp/tmp.Q1NHZnTJr2 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-564f9bc786-mpbk6 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qHbk2ae6eN pod "percona-xtradb-cluster-operator-564f9bc786-mpbk6" deleted from pxc-operator namespace + cat /tmp/tmp.Q1NHZnTJr2 + rm /tmp/tmp.qHbk2ae6eN /tmp/tmp.Q1NHZnTJr2 + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.7cthriRvq7 ++ mktemp + local LAST_ERR=/tmp/tmp.45BkaFnecs + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7cthriRvq7 perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-13972 namespace + cat /tmp/tmp.45BkaFnecs + rm /tmp/tmp.7cthriRvq7 /tmp/tmp.45BkaFnecs + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.H1Vcu3SMKz ++ mktemp + local LAST_ERR=/tmp/tmp.OAbTqLZYvd + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.H1Vcu3SMKz deployment.apps "pxc-client" deleted from smart-update2-13972 namespace + cat /tmp/tmp.OAbTqLZYvd + rm /tmp/tmp.H1Vcu3SMKz /tmp/tmp.OAbTqLZYvd + return 0 + sleep 30 + kubectl_bin delete pod -l run=version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.aJTFy7tUdh ++ mktemp + local LAST_ERR=/tmp/tmp.Pt7BOyWsf0 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pod -l run=version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.aJTFy7tUdh pod "version-service-cr-58b8ffbbb4-s6nkp" deleted from smart-update2-13972 namespace + cat /tmp/tmp.Pt7BOyWsf0 + rm /tmp/tmp.aJTFy7tUdh /tmp/tmp.Pt7BOyWsf0 + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.17v1R4IhKb ++ mktemp + local LAST_ERR=/tmp/tmp.LQsdLFLI37 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.17v1R4IhKb pod "version-service-66774c865b-2mtdw" deleted from smart-update2-13972 namespace + cat /tmp/tmp.LQsdLFLI37 + rm /tmp/tmp.17v1R4IhKb /tmp/tmp.LQsdLFLI37 + return 0 + check_telemetry_transfer http://version-service-cr.smart-update2-13972.svc.cluster.local:11000 disabled disabled + local cr_vs_uri=http://version-service-cr.smart-update2-13972.svc.cluster.local:11000 + local cr_vs_channel=disabled + local telemetry_state=disabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.0VphMfB7FH ++ mktemp + local LAST_ERR=/tmp/tmp.DSlajvUBSd + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0VphMfB7FH deployment.apps/pxc-client created + cat /tmp/tmp.DSlajvUBSd + rm /tmp/tmp.0VphMfB7FH /tmp/tmp.DSlajvUBSd + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml + kubectl_bin apply -f - + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' ++ mktemp + local LAST_OUT=/tmp/tmp.a5JMiftbYT ++ mktemp + local LAST_ERR=/tmp/tmp.WXlxb24CZn + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.a5JMiftbYT secret/minimal-cluster unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.WXlxb24CZn + rm /tmp/tmp.a5JMiftbYT /tmp/tmp.WXlxb24CZn + return 0 + yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"' + kubectl_bin apply -f - + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420"' + yq eval '.spec.crVersion="9.9.9"' + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.41-32.1"' + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + yq eval '.spec.upgradeOptions.apply="disabled"' + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-13972.svc.cluster.local:11000"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cr-minimal.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.znbdaDjfUn ++ mktemp + local LAST_ERR=/tmp/tmp.Kr8yEDvu78 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.znbdaDjfUn perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.Kr8yEDvu78 + rm /tmp/tmp.znbdaDjfUn /tmp/tmp.Kr8yEDvu78 + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-pxc ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-pxc ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in $(seq 0 $last_pod) + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo minimal-cluster-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rq3hOVan0L ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iUcFRTq8DJ +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.rq3hOVan0L +++ cat /tmp/tmp.iUcFRTq8DJ +++ rm /tmp/tmp.rq3hOVan0L /tmp/tmp.iUcFRTq8DJ +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + local size=1 + local last_pod + local max_retry=480 + [[ -z minimal-cluster-haproxy ]] + [[ 1 -le 0 ]] + last_pod=0 + desc 'Waiting for 1 running pods for minimal-cluster-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 1 running pods for minimal-cluster-haproxy ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in $(seq 0 $last_pod) + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo minimal-cluster-haproxy-0 ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mdw8YMZGDY +++ mktemp ++ local LAST_ERR=/tmp/tmp.R5NT61p3N6 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mdw8YMZGDY ++ cat /tmp/tmp.R5NT61p3N6 ++ rm /tmp/tmp.mdw8YMZGDY /tmp/tmp.R5NT61p3N6 ++ return 0 + client_pod=pxc-client-6fc75f888d-sqmwb + wait_pod pxc-client-6fc75f888d-sqmwb + local pod=pxc-client-6fc75f888d-sqmwb + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6fc75f888d-sqmwb ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-sqmwb condition met waiting for pod/pxc-client-6fc75f888d-sqmwb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.e0e76ACEWe +++ mktemp ++ local LAST_ERR=/tmp/tmp.5FlHSUyz0r ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.e0e76ACEWe ++ cat /tmp/tmp.5FlHSUyz0r ++ rm /tmp/tmp.e0e76ACEWe /tmp/tmp.5FlHSUyz0r ++ return 0 + client_pod=pxc-client-6fc75f888d-sqmwb + wait_pod pxc-client-6fc75f888d-sqmwb + local pod=pxc-client-6fc75f888d-sqmwb + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6fc75f888d-sqmwb + local container= + set +o xtrace pod/pxc-client-6fc75f888d-sqmwb condition met waiting for pod/pxc-client-6fc75f888d-sqmwb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.request.content".msg.customResourceUid)' + grep -Eo '\{.*\}' + grep -E 'server request payload|unary call' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-cr-58b8ffbbb4-pq6ng ++ mktemp + local LAST_OUT=/tmp/tmp.jn9a66VGAq ++ mktemp + local LAST_ERR=/tmp/tmp.pKOIHVYvB3 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs version-service-cr-58b8ffbbb4-pq6ng + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jn9a66VGAq + cat /tmp/tmp.pKOIHVYvB3 + rm /tmp/tmp.jn9a66VGAq /tmp/tmp.pKOIHVYvB3 + return 0 + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.request.content".msg.kubeVersion)' + grep -Eo '\{.*\}' + jq 'del(."grpc.start_time")' + grep -E 'server request payload|unary call' + jq 'del(."grpc.time_ms")' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-66774c865b-tvcfd ++ mktemp + local LAST_OUT=/tmp/tmp.JQOpfT7g1y ++ mktemp + local LAST_ERR=/tmp/tmp.B1QbR9mO3Q + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs version-service-66774c865b-tvcfd + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JQOpfT7g1y + cat /tmp/tmp.B1QbR9mO3Q + rm /tmp/tmp.JQOpfT7g1y /tmp/tmp.B1QbR9mO3Q + return 0 + local telemetry_log_file=disabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' disabled == disabled -a disabled == enabled ']' + local image_prefix=disabled + image_prefix=disabled + local telemetry_cr_log_file=disabled_telemetry.version-service-cr-disabled-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + [[ disabled == \d\i\s\a\b\l\e\d\-\r\e\c\o\m\m\e\n\d\e\d ]] + [[ disabled == \d\i\s\a\b\l\e\d\-\l\a\t\e\s\t ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' disabled == disabled -a disabled == disabled ']' + desc 'CR VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- CR VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.WkA2TS2tBT/disabled_telemetry.version-service-cr.log.json ]] + desc 'operator VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.WkA2TS2tBT/disabled_telemetry.version-service.log.json ]] + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.1VsouCmLQL ++ mktemp + local LAST_ERR=/tmp/tmp.zBq3FZ0fqP + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1VsouCmLQL perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.zBq3FZ0fqP + rm /tmp/tmp.1VsouCmLQL /tmp/tmp.zBq3FZ0fqP + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.VbWc2bIb7P +++ mktemp ++ local LAST_ERR=/tmp/tmp.VaiI15Dagc ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VbWc2bIb7P ++ cat /tmp/tmp.VaiI15Dagc ++ rm /tmp/tmp.VbWc2bIb7P /tmp/tmp.VaiI15Dagc ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-564f9bc786-9sdvf ++ mktemp + local LAST_OUT=/tmp/tmp.k131o5OTSg ++ mktemp + local LAST_ERR=/tmp/tmp.nblev05Us0 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-564f9bc786-9sdvf + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.k131o5OTSg pod "percona-xtradb-cluster-operator-564f9bc786-9sdvf" deleted from pxc-operator namespace + cat /tmp/tmp.nblev05Us0 + rm /tmp/tmp.k131o5OTSg /tmp/tmp.nblev05Us0 + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.JKnC3mx2Is ++ mktemp + local LAST_ERR=/tmp/tmp.BSonlXDc50 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JKnC3mx2Is perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-13972 namespace + cat /tmp/tmp.BSonlXDc50 + rm /tmp/tmp.JKnC3mx2Is /tmp/tmp.BSonlXDc50 + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.NudoLIbvlZ ++ mktemp + local LAST_ERR=/tmp/tmp.O7IG3FxH2G + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NudoLIbvlZ deployment.apps "pxc-client" deleted from smart-update2-13972 namespace + cat /tmp/tmp.O7IG3FxH2G + rm /tmp/tmp.NudoLIbvlZ /tmp/tmp.O7IG3FxH2G + return 0 + sleep 30 + kubectl_bin delete deployment version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.eviTYWDvlt ++ mktemp + local LAST_ERR=/tmp/tmp.LvWE4g0nEU + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete deployment version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eviTYWDvlt deployment.apps "version-service-cr" deleted from smart-update2-13972 namespace + cat /tmp/tmp.LvWE4g0nEU + rm /tmp/tmp.eviTYWDvlt /tmp/tmp.LvWE4g0nEU + return 0 + desc 'Telemetry testing finished' + set +o xtrace ----------------------------------------------------------------------------------- Telemetry testing finished ----------------------------------------------------------------------------------- + desc 'PXC cluster with version service offline' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster with version service offline ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/smart-update-version-service-unreachable.yml /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420"' /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + spinup_pxc smart-update /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.yTYrof3jKS ++ mktemp + local LAST_ERR=/tmp/tmp.pFMRpBUu8d + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yTYrof3jKS secret/my-cluster-secrets created secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.pFMRpBUu8d + rm /tmp/tmp.yTYrof3jKS /tmp/tmp.pFMRpBUu8d + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-13972~ + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local LAST_OUT=/tmp/tmp.hLrBiXuYak ++ mktemp + local LAST_ERR=/tmp/tmp.3XUIq2GmZb + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hLrBiXuYak deployment.apps/pxc-client created + cat /tmp/tmp.3XUIq2GmZb + rm /tmp/tmp.hLrBiXuYak /tmp/tmp.3XUIq2GmZb + return 0 + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + apply_config /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local config_file=/tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml '' + local input_file=/tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + cat /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local LAST_OUT=/tmp/tmp.nFG6TQPOCp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-13972~ + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ mktemp + local LAST_ERR=/tmp/tmp.8i7DKsYLOc + local exit_status=0 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nFG6TQPOCp perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.8i7DKsYLOc + rm /tmp/tmp.nFG6TQPOCp /tmp/tmp.8i7DKsYLOc + return 0 + desc 'waiting pods to be running for smart-update' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for smart-update ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.91Esqje0Cm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RvjxfeCtkT +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.91Esqje0Cm +++ cat /tmp/tmp.RvjxfeCtkT +++ rm /tmp/tmp.91Esqje0Cm /tmp/tmp.RvjxfeCtkT +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy ++ get_proxy_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vysWUTv6Zj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lcZROqI3oN +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.vysWUTv6Zj +++ cat /tmp/tmp.lcZROqI3oN +++ rm /tmp/tmp.vysWUTv6Zj /tmp/tmp.lcZROqI3oN +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.POvbBEsF5N ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SY7rBnkEpy +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.POvbBEsF5N +++ cat /tmp/tmp.SY7rBnkEpy +++ rm /tmp/tmp.POvbBEsF5N /tmp/tmp.SY7rBnkEpy +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z smart-update-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for smart-update-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for smart-update-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in $(seq 0 $last_pod) + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-pxc-0 + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-2 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + wait_for_running smart-update-haproxy 2 + local name=smart-update-haproxy + local size=2 + local last_pod + local max_retry=480 + [[ -z smart-update-haproxy ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for smart-update-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for smart-update-haproxy ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in $(seq 0 $last_pod) + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-haproxy-0 + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met waiting for pod/smart-update-haproxy-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod smart-update-haproxy-1 480 + local pod=smart-update-haproxy-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-haproxy-1 ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-1 condition met waiting for pod/smart-update-haproxy-1 to become Ready.Ok + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FlqXQNxRRg +++ mktemp ++ local LAST_ERR=/tmp/tmp.h6hCQAM0ky ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FlqXQNxRRg ++ cat /tmp/tmp.h6hCQAM0ky ++ rm /tmp/tmp.FlqXQNxRRg /tmp/tmp.h6hCQAM0ky ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Bfw70ZEct8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.YBmCwRK4bz ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Bfw70ZEct8 ++ cat /tmp/tmp.YBmCwRK4bz ++ rm /tmp/tmp.Bfw70ZEct8 /tmp/tmp.YBmCwRK4bz ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.XbE31cPnHw ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.XHyezDXlov +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.XbE31cPnHw +++++ cat /tmp/tmp.XHyezDXlov +++++ rm /tmp/tmp.XbE31cPnHw /tmp/tmp.XHyezDXlov +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.B2TPo9b2Xd +++ mktemp ++ local LAST_ERR=/tmp/tmp.kUr8pYbW12 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.B2TPo9b2Xd ++ cat /tmp/tmp.kUr8pYbW12 ++ rm /tmp/tmp.B2TPo9b2Xd /tmp/tmp.kUr8pYbW12 ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OKLp77zu4Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.6xckhP1X46 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OKLp77zu4Q ++ cat /tmp/tmp.6xckhP1X46 ++ rm /tmp/tmp.OKLp77zu4Q /tmp/tmp.6xckhP1X46 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6RSj3DwHWF +++ mktemp ++ local LAST_ERR=/tmp/tmp.6WBQgcEIc1 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6RSj3DwHWF ++ cat /tmp/tmp.6WBQgcEIc1 ++ rm /tmp/tmp.6RSj3DwHWF /tmp/tmp.6WBQgcEIc1 ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6df8986959-zblv5 + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data smart-update 100500 no-proxy -P3306 + local cluster=smart-update + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Qc4FHs11Lo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6NL44Cv8yX +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Qc4FHs11Lo +++ cat /tmp/tmp.6NL44Cv8yX +++ rm /tmp/tmp.Qc4FHs11Lo /tmp/tmp.6NL44Cv8yX +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.i3RiAuxz9c +++ mktemp ++ local LAST_ERR=/tmp/tmp.0cddwuHLvY ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.i3RiAuxz9c ++ cat /tmp/tmp.0cddwuHLvY ++ rm /tmp/tmp.i3RiAuxz9c /tmp/tmp.0cddwuHLvY ++ return 0 + root_pass=root_password ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LnVTklGxIA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jdm9N1KVE3 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.LnVTklGxIA +++ cat /tmp/tmp.jdm9N1KVE3 +++ rm /tmp/tmp.LnVTklGxIA /tmp/tmp.jdm9N1KVE3 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + proxy=smart-update-haproxy + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h smart-update-pxc-0.smart-update-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LOq3w20lPg +++ mktemp ++ local LAST_ERR=/tmp/tmp.vOf7GqENMn ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LOq3w20lPg ++ cat /tmp/tmp.vOf7GqENMn ++ rm /tmp/tmp.LOq3w20lPg /tmp/tmp.vOf7GqENMn ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6df8986959-zblv5 + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data smart-update select-1 no-proxy -P3306 + local cluster=smart-update + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pQoiiJMftJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZCjyA4sLbz +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.pQoiiJMftJ +++ cat /tmp/tmp.ZCjyA4sLbz +++ rm /tmp/tmp.pQoiiJMftJ /tmp/tmp.ZCjyA4sLbz +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VEXFfqIGxZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.85wwayautm ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VEXFfqIGxZ ++ cat /tmp/tmp.85wwayautm ++ rm /tmp/tmp.VEXFfqIGxZ /tmp/tmp.85wwayautm ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6Vt8Dg42Bi ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JdD4wEnQBT +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.6Vt8Dg42Bi +++ cat /tmp/tmp.JdD4wEnQBT +++ rm /tmp/tmp.6Vt8Dg42Bi /tmp/tmp.JdD4wEnQBT +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OZJuseZd8d +++ mktemp ++ local LAST_ERR=/tmp/tmp.hXa1NwaV6e ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OZJuseZd8d ++ cat /tmp/tmp.hXa1NwaV6e ++ rm /tmp/tmp.OZJuseZd8d /tmp/tmp.hXa1NwaV6e ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.A83OVoAIPH +++ mktemp ++ local LAST_ERR=/tmp/tmp.jWCKz4i4WK ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A83OVoAIPH ++ cat /tmp/tmp.jWCKz4i4WK ++ rm /tmp/tmp.A83OVoAIPH /tmp/tmp.jWCKz4i4WK ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0RAPfHDKVA +++ mktemp ++ local LAST_ERR=/tmp/tmp.TJylXanfZm ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0RAPfHDKVA ++ cat /tmp/tmp.TJylXanfZm ++ rm /tmp/tmp.0RAPfHDKVA /tmp/tmp.TJylXanfZm ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ echo pxc-client-6df8986959-zblv5 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + kubectl exec smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + grep -E -o 'early-plugin-load=keyring_\w+.so' + return 1 + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AD2SwLsGxC +++ mktemp ++ local LAST_ERR=/tmp/tmp.qxm2idmL0L ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AD2SwLsGxC ++ cat /tmp/tmp.qxm2idmL0L ++ rm /tmp/tmp.AD2SwLsGxC /tmp/tmp.qxm2idmL0L ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tmA0kS1ygY +++ mktemp ++ local LAST_ERR=/tmp/tmp.2us85iBlju ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tmA0kS1ygY ++ cat /tmp/tmp.2us85iBlju ++ rm /tmp/tmp.tmA0kS1ygY /tmp/tmp.2us85iBlju ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.VJ53rx1FyR ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.xFJ76ZVlEc +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.VJ53rx1FyR +++++ cat /tmp/tmp.xFJ76ZVlEc +++++ rm /tmp/tmp.VJ53rx1FyR /tmp/tmp.xFJ76ZVlEc +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ERNappbFh3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.LyUhlF0Jr6 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ERNappbFh3 ++ cat /tmp/tmp.LyUhlF0Jr6 ++ rm /tmp/tmp.ERNappbFh3 /tmp/tmp.LyUhlF0Jr6 ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ kubectl_bin get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bRKnc3Elxy +++ mktemp ++ local LAST_ERR=/tmp/tmp.GyNmjeifYE ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bRKnc3Elxy ++ cat /tmp/tmp.GyNmjeifYE ++ rm /tmp/tmp.bRKnc3Elxy /tmp/tmp.GyNmjeifYE ++ return 0 + [[ percona/percona-xtradb-cluster:8.0.41-32.1 != \p\e\r\c\o\n\a\/\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\:\8\.\0\.\4\1\-\3\2\.\1 ]] + desc 'PXC cluster update with recommended image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with recommended image by version service ----------------------------------------------------------------------------------- + vs_image=recommended ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UWrA2udpqX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TZfK44rRiq +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.UWrA2udpqX +++ cat /tmp/tmp.TZfK44rRiq +++ rm /tmp/tmp.UWrA2udpqX /tmp/tmp.TZfK44rRiq +++ return 0 ++ client_pod=pxc-client-6df8986959-zblv5 ++ wait_pod pxc-client-6df8986959-zblv5 ++ local pod=pxc-client-6df8986959-zblv5 ++ local max_retry=480 ++ local ns= +++ grep -E '^(pxc|proxysql)$' +++ echo pxc-client-6df8986959-zblv5 +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ local container= ++ set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.txq1rGcdxQ ++ mktemp + local LAST_ERR=/tmp/tmp.NDiRwZkyIL + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.txq1rGcdxQ perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.NDiRwZkyIL + rm /tmp/tmp.txq1rGcdxQ /tmp/tmp.NDiRwZkyIL + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pibxWP8F0o +++ mktemp ++ local LAST_ERR=/tmp/tmp.uAlhs7SIFo ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pibxWP8F0o ++ cat /tmp/tmp.uAlhs7SIFo ++ rm /tmp/tmp.pibxWP8F0o /tmp/tmp.uAlhs7SIFo ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qMvvPWyWsK +++ mktemp ++ local LAST_ERR=/tmp/tmp.N4KpyJjyOE ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qMvvPWyWsK ++ cat /tmp/tmp.N4KpyJjyOE ++ rm /tmp/tmp.qMvvPWyWsK /tmp/tmp.N4KpyJjyOE ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.ilXwVNRmcj ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.T7EMHGGy78 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.ilXwVNRmcj +++++ cat /tmp/tmp.T7EMHGGy78 +++++ rm /tmp/tmp.ilXwVNRmcj /tmp/tmp.T7EMHGGy78 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hbUDz1FVgq +++ mktemp ++ local LAST_ERR=/tmp/tmp.YpyWriQBa5 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hbUDz1FVgq ++ cat /tmp/tmp.YpyWriQBa5 ++ rm /tmp/tmp.hbUDz1FVgq /tmp/tmp.YpyWriQBa5 ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ seq 0 2 + for i in $(seq 0 $((CLUSTER_SIZE - 1))) + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DSCU0DnUVh +++ mktemp ++ local LAST_ERR=/tmp/tmp.rE4T85y9LQ ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DSCU0DnUVh ++ cat /tmp/tmp.rE4T85y9LQ ++ rm /tmp/tmp.DSCU0DnUVh /tmp/tmp.rE4T85y9LQ ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6df8986959-zblv5 ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + for i in $(seq 0 $((CLUSTER_SIZE - 1))) + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UWkWhZkq8Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.q9x7eM0Jsp ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UWkWhZkq8Z ++ cat /tmp/tmp.q9x7eM0Jsp ++ rm /tmp/tmp.UWkWhZkq8Z /tmp/tmp.q9x7eM0Jsp ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + for i in $(seq 0 $((CLUSTER_SIZE - 1))) + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bwc1PPfeQh +++ mktemp ++ local LAST_ERR=/tmp/tmp.kpsy9kRvEB ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bwc1PPfeQh ++ cat /tmp/tmp.kpsy9kRvEB ++ rm /tmp/tmp.bwc1PPfeQh /tmp/tmp.kpsy9kRvEB ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6df8986959-zblv5 + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + kubectl_bin delete -f /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.0rUjSceaW9 ++ mktemp + local LAST_ERR=/tmp/tmp.0HzqhZ1Agr + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0rUjSceaW9 perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-13972 namespace + cat /tmp/tmp.0HzqhZ1Agr + rm /tmp/tmp.0rUjSceaW9 /tmp/tmp.0HzqhZ1Agr + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.tUJ5dwxeHS ++ mktemp + local LAST_ERR=/tmp/tmp.Zx4YqwfEfu + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tUJ5dwxeHS persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-13972 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-13972 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-13972 namespace + cat /tmp/tmp.Zx4YqwfEfu + rm /tmp/tmp.tUJ5dwxeHS /tmp/tmp.Zx4YqwfEfu + return 0 + desc 'PXC cluster update with the latest image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with the latest image by version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.5FAHxo8Nyp ++ mktemp + local LAST_ERR=/tmp/tmp.36Lhz2WvOi + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5FAHxo8Nyp secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.36Lhz2WvOi + rm /tmp/tmp.5FAHxo8Nyp /tmp/tmp.36Lhz2WvOi + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ mktemp + local LAST_OUT=/tmp/tmp.fEhsjcS9iH + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-13972~ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + local LAST_ERR=/tmp/tmp.DL95NVfWz0 + local exit_status=0 + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fEhsjcS9iH deployment.apps/pxc-client unchanged + cat /tmp/tmp.DL95NVfWz0 + rm /tmp/tmp.fEhsjcS9iH /tmp/tmp.DL95NVfWz0 + return 0 + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + apply_config /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local config_file=/tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local pvc_name= + '[' -z '' ']' + cat_config /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml '' + local input_file=/tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local pvc_name= + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-13972~ + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + cat /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_OUT=/tmp/tmp.agR0oa8IzN ++ mktemp + local LAST_ERR=/tmp/tmp.Jg0VVgqfz6 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.agR0oa8IzN perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.Jg0VVgqfz6 + rm /tmp/tmp.agR0oa8IzN /tmp/tmp.Jg0VVgqfz6 + return 0 + desc 'waiting pods to be running for smart-update' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for smart-update ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.evUfprPaCQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zNEX7vaV0o +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.evUfprPaCQ +++ cat /tmp/tmp.zNEX7vaV0o +++ rm /tmp/tmp.evUfprPaCQ /tmp/tmp.zNEX7vaV0o +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy ++ get_proxy_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Y8lJuY4miu ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zCVQitulcl +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Y8lJuY4miu +++ cat /tmp/tmp.zCVQitulcl +++ rm /tmp/tmp.Y8lJuY4miu /tmp/tmp.zCVQitulcl +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sc4icnRbLt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VIaGVttv8P +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.sc4icnRbLt +++ cat /tmp/tmp.VIaGVttv8P +++ rm /tmp/tmp.sc4icnRbLt /tmp/tmp.VIaGVttv8P +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z smart-update-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for smart-update-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for smart-update-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in $(seq 0 $last_pod) + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + wait_for_running smart-update-haproxy 2 + local name=smart-update-haproxy + local size=2 + local last_pod + local max_retry=480 + [[ -z smart-update-haproxy ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for smart-update-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for smart-update-haproxy ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in $(seq 0 $last_pod) + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-haproxy-0 + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met waiting for pod/smart-update-haproxy-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod smart-update-haproxy-1 480 + local pod=smart-update-haproxy-1 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-1 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/smart-update-haproxy-1 condition met waiting for pod/smart-update-haproxy-1 to become Ready.Ok + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w4eSMEboMy +++ mktemp ++ local LAST_ERR=/tmp/tmp.oKdu00DhFP ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.w4eSMEboMy ++ cat /tmp/tmp.oKdu00DhFP ++ rm /tmp/tmp.w4eSMEboMy /tmp/tmp.oKdu00DhFP ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NvpPiGTGdm +++ mktemp ++ local LAST_ERR=/tmp/tmp.pQWJWSply8 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NvpPiGTGdm ++ cat /tmp/tmp.pQWJWSply8 ++ rm /tmp/tmp.NvpPiGTGdm /tmp/tmp.pQWJWSply8 ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.BVz9j6sD1w ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.wdqikSv946 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.BVz9j6sD1w +++++ cat /tmp/tmp.wdqikSv946 +++++ rm /tmp/tmp.BVz9j6sD1w /tmp/tmp.wdqikSv946 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pM3SAB8taM +++ mktemp ++ local LAST_ERR=/tmp/tmp.BR7qixLhA9 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pM3SAB8taM ++ cat /tmp/tmp.BR7qixLhA9 ++ rm /tmp/tmp.pM3SAB8taM /tmp/tmp.BR7qixLhA9 ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.p7hvSDtciz +++ mktemp ++ local LAST_ERR=/tmp/tmp.g53VxW7kJp ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.p7hvSDtciz ++ cat /tmp/tmp.g53VxW7kJp ++ rm /tmp/tmp.p7hvSDtciz /tmp/tmp.g53VxW7kJp ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DrRoM53S6f +++ mktemp ++ local LAST_ERR=/tmp/tmp.4aNHSJjBXl ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DrRoM53S6f ++ cat /tmp/tmp.4aNHSJjBXl ++ rm /tmp/tmp.DrRoM53S6f /tmp/tmp.4aNHSJjBXl ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data smart-update 100500 no-proxy -P3306 + local cluster=smart-update + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jxjgeZCeND ++++ mktemp +++ local LAST_ERR=/tmp/tmp.94VUIH7Xaj +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.jxjgeZCeND +++ cat /tmp/tmp.94VUIH7Xaj +++ rm /tmp/tmp.jxjgeZCeND /tmp/tmp.94VUIH7Xaj +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Wu2BpdWRsh +++ mktemp ++ local LAST_ERR=/tmp/tmp.zXEPZUWGcl ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Wu2BpdWRsh ++ cat /tmp/tmp.zXEPZUWGcl ++ rm /tmp/tmp.Wu2BpdWRsh /tmp/tmp.zXEPZUWGcl ++ return 0 + root_pass=root_password ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LODga7YIdL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1ZBFuPsb5k +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.LODga7YIdL +++ cat /tmp/tmp.1ZBFuPsb5k +++ rm /tmp/tmp.LODga7YIdL /tmp/tmp.1ZBFuPsb5k +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + proxy=smart-update-haproxy + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h smart-update-pxc-0.smart-update-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ompgvldwBj +++ mktemp ++ local LAST_ERR=/tmp/tmp.XzkWEvh5ly ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ompgvldwBj ++ cat /tmp/tmp.XzkWEvh5ly ++ rm /tmp/tmp.ompgvldwBj /tmp/tmp.XzkWEvh5ly ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data smart-update select-1 no-proxy -P3306 + local cluster=smart-update + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.froHbda20V ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RonOP062rI +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.froHbda20V +++ cat /tmp/tmp.RonOP062rI +++ rm /tmp/tmp.froHbda20V /tmp/tmp.RonOP062rI +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.op8oyegZuU +++ mktemp ++ local LAST_ERR=/tmp/tmp.uEyYV6xMtZ ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.op8oyegZuU ++ cat /tmp/tmp.uEyYV6xMtZ ++ rm /tmp/tmp.op8oyegZuU /tmp/tmp.uEyYV6xMtZ ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PR2A37gVKH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FNuYEb70Rq +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.PR2A37gVKH +++ cat /tmp/tmp.FNuYEb70Rq +++ rm /tmp/tmp.PR2A37gVKH /tmp/tmp.FNuYEb70Rq +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Qc7K2HmXeK +++ mktemp ++ local LAST_ERR=/tmp/tmp.B5qZlqE9Kn ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Qc7K2HmXeK ++ cat /tmp/tmp.B5qZlqE9Kn ++ rm /tmp/tmp.Qc7K2HmXeK /tmp/tmp.B5qZlqE9Kn ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yKSkhoWOU2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Q3odxFgxsA ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yKSkhoWOU2 ++ cat /tmp/tmp.Q3odxFgxsA ++ rm /tmp/tmp.yKSkhoWOU2 /tmp/tmp.Q3odxFgxsA ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ echo pxc-client-6df8986959-zblv5 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gkt2aaHP4z +++ mktemp ++ local LAST_ERR=/tmp/tmp.9NKnRFOdR1 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gkt2aaHP4z ++ cat /tmp/tmp.9NKnRFOdR1 ++ rm /tmp/tmp.gkt2aaHP4z /tmp/tmp.9NKnRFOdR1 ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + grep -E -o 'early-plugin-load=keyring_\w+.so' + kubectl exec smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + return 1 + vs_image=latest ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.m7G9GQ0H95 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TyNx39ZlWu +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.m7G9GQ0H95 +++ cat /tmp/tmp.TyNx39ZlWu +++ rm /tmp/tmp.m7G9GQ0H95 /tmp/tmp.TyNx39ZlWu +++ return 0 ++ client_pod=pxc-client-6df8986959-zblv5 ++ wait_pod pxc-client-6df8986959-zblv5 ++ local pod=pxc-client-6df8986959-zblv5 ++ local max_retry=480 ++ local ns= +++ grep -E '^(pxc|proxysql)$' +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ echo pxc-client-6df8986959-zblv5 ++ local container= ++ set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.Eh2K3H8JBM ++ mktemp + local LAST_ERR=/tmp/tmp.Zl3IKel9X3 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Eh2K3H8JBM perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.Zl3IKel9X3 + rm /tmp/tmp.Eh2K3H8JBM /tmp/tmp.Zl3IKel9X3 + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aZUUHdMUc1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZbA7CtsP0q ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aZUUHdMUc1 ++ cat /tmp/tmp.ZbA7CtsP0q ++ rm /tmp/tmp.aZUUHdMUc1 /tmp/tmp.ZbA7CtsP0q ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.D0YDxSaS45 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4NnHxU5hjp ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.D0YDxSaS45 ++ cat /tmp/tmp.4NnHxU5hjp ++ rm /tmp/tmp.D0YDxSaS45 /tmp/tmp.4NnHxU5hjp ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.lzubqnXWEG ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.okRwmKIq7F +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.lzubqnXWEG +++++ cat /tmp/tmp.okRwmKIq7F +++++ rm /tmp/tmp.lzubqnXWEG /tmp/tmp.okRwmKIq7F +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KhSO1SAUaW +++ mktemp ++ local LAST_ERR=/tmp/tmp.pejQf3uzAW ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KhSO1SAUaW ++ cat /tmp/tmp.pejQf3uzAW ++ rm /tmp/tmp.KhSO1SAUaW /tmp/tmp.pejQf3uzAW ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ seq 0 2 + for i in $(seq 0 $((CLUSTER_SIZE - 1))) + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5wK5BRaFPG +++ mktemp ++ local LAST_ERR=/tmp/tmp.F2Itjhgo9u ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5wK5BRaFPG ++ cat /tmp/tmp.F2Itjhgo9u ++ rm /tmp/tmp.5wK5BRaFPG /tmp/tmp.F2Itjhgo9u ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6df8986959-zblv5 ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + for i in $(seq 0 $((CLUSTER_SIZE - 1))) + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qqUnRWf8Bq +++ mktemp ++ local LAST_ERR=/tmp/tmp.4SJ4ArVKZM ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qqUnRWf8Bq ++ cat /tmp/tmp.4SJ4ArVKZM ++ rm /tmp/tmp.qqUnRWf8Bq /tmp/tmp.4SJ4ArVKZM ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + for i in $(seq 0 $((CLUSTER_SIZE - 1))) + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TBwgDM4ZIK +++ mktemp ++ local LAST_ERR=/tmp/tmp.Bi7FiOWskh ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TBwgDM4ZIK ++ cat /tmp/tmp.Bi7FiOWskh ++ rm /tmp/tmp.TBwgDM4ZIK /tmp/tmp.Bi7FiOWskh ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + kubectl_bin delete -f /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.78fU3mutr7 ++ mktemp + local LAST_ERR=/tmp/tmp.HBz6FoNgL7 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.78fU3mutr7 perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-13972 namespace + cat /tmp/tmp.HBz6FoNgL7 + rm /tmp/tmp.78fU3mutr7 /tmp/tmp.HBz6FoNgL7 + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.nr77TFwfFL ++ mktemp + local LAST_ERR=/tmp/tmp.ORc7CviIwz + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nr77TFwfFL persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-13972 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-13972 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-13972 namespace + cat /tmp/tmp.ORc7CviIwz + rm /tmp/tmp.nr77TFwfFL /tmp/tmp.ORc7CviIwz + return 0 + desc 'PXC cluster update with explicitly specified image inside version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with explicitly specified image inside version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.jToa1448rt ++ mktemp + local LAST_ERR=/tmp/tmp.KBRywpRldg + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jToa1448rt secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.KBRywpRldg + rm /tmp/tmp.jToa1448rt /tmp/tmp.KBRywpRldg + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-13972~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + local LAST_OUT=/tmp/tmp.hQ4DpKWISy + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' ++ mktemp + local LAST_ERR=/tmp/tmp.DBgXamQ5bZ + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hQ4DpKWISy deployment.apps/pxc-client unchanged + cat /tmp/tmp.DBgXamQ5bZ + rm /tmp/tmp.hQ4DpKWISy /tmp/tmp.DBgXamQ5bZ + return 0 + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + apply_config /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local config_file=/tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local pvc_name= + '[' -z '' ']' + cat_config /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml '' + local input_file=/tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + local pvc_name= + kubectl_bin apply -f - + cat /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-13972~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + local LAST_OUT=/tmp/tmp.KZok64g2bR ++ mktemp + local LAST_ERR=/tmp/tmp.mPKUMQGZf8 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KZok64g2bR perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.mPKUMQGZf8 + rm /tmp/tmp.KZok64g2bR /tmp/tmp.mPKUMQGZf8 + return 0 + desc 'waiting pods to be running for smart-update' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for smart-update ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pHgfReXLZc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GBuSb4Mof5 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.pHgfReXLZc +++ cat /tmp/tmp.GBuSb4Mof5 +++ rm /tmp/tmp.pHgfReXLZc /tmp/tmp.GBuSb4Mof5 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy ++ get_proxy_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hDbd8JKaxs ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ijV0NeJH22 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.hDbd8JKaxs +++ cat /tmp/tmp.ijV0NeJH22 +++ rm /tmp/tmp.hDbd8JKaxs /tmp/tmp.ijV0NeJH22 +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qw1Y44EpOU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TBnn4yWOKw +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.qw1Y44EpOU +++ cat /tmp/tmp.TBnn4yWOKw +++ rm /tmp/tmp.qw1Y44EpOU /tmp/tmp.TBnn4yWOKw +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z smart-update-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for smart-update-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for smart-update-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in $(seq 0 $last_pod) + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-0 + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-1 + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-2 + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + wait_for_running smart-update-haproxy 2 + local name=smart-update-haproxy + local size=2 + local last_pod + local max_retry=480 + [[ -z smart-update-haproxy ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for smart-update-haproxy' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for smart-update-haproxy ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in $(seq 0 $last_pod) + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met waiting for pod/smart-update-haproxy-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod smart-update-haproxy-1 480 + local pod=smart-update-haproxy-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-haproxy-1 ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-1 condition met waiting for pod/smart-update-haproxy-1 to become Ready.Ok + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sHr1CqHHDN +++ mktemp ++ local LAST_ERR=/tmp/tmp.Mzaxg5zWIx ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sHr1CqHHDN ++ cat /tmp/tmp.Mzaxg5zWIx ++ rm /tmp/tmp.sHr1CqHHDN /tmp/tmp.Mzaxg5zWIx ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ouEYjkuxhZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.zvEIssvOsH ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ouEYjkuxhZ ++ cat /tmp/tmp.zvEIssvOsH ++ rm /tmp/tmp.ouEYjkuxhZ /tmp/tmp.zvEIssvOsH ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.2RfZ8gru4C ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.5p3rd9mz25 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.2RfZ8gru4C +++++ cat /tmp/tmp.5p3rd9mz25 +++++ rm /tmp/tmp.2RfZ8gru4C /tmp/tmp.5p3rd9mz25 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6iazUixBfg +++ mktemp ++ local LAST_ERR=/tmp/tmp.MABLM1cGfW ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6iazUixBfg ++ cat /tmp/tmp.MABLM1cGfW ++ rm /tmp/tmp.6iazUixBfg /tmp/tmp.MABLM1cGfW ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.3Nk9qv4LAQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.bAk10VOR9v ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3Nk9qv4LAQ ++ cat /tmp/tmp.bAk10VOR9v ++ rm /tmp/tmp.3Nk9qv4LAQ /tmp/tmp.bAk10VOR9v ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CP0uIY3O0z +++ mktemp ++ local LAST_ERR=/tmp/tmp.iWBo05JV9I ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CP0uIY3O0z ++ cat /tmp/tmp.iWBo05JV9I ++ rm /tmp/tmp.CP0uIY3O0z /tmp/tmp.iWBo05JV9I ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6df8986959-zblv5 + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data smart-update 100500 no-proxy -P3306 + local cluster=smart-update + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tlX34SZrHO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sjdEmPQnsZ +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.tlX34SZrHO +++ cat /tmp/tmp.sjdEmPQnsZ +++ rm /tmp/tmp.tlX34SZrHO /tmp/tmp.sjdEmPQnsZ +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.kvGhYQS33F +++ mktemp ++ local LAST_ERR=/tmp/tmp.m78C7abQiK ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kvGhYQS33F ++ cat /tmp/tmp.m78C7abQiK ++ rm /tmp/tmp.kvGhYQS33F /tmp/tmp.m78C7abQiK ++ return 0 + root_pass=root_password ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jo4TG2iv2w ++++ mktemp +++ local LAST_ERR=/tmp/tmp.e2AQtfgoiD +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.jo4TG2iv2w +++ cat /tmp/tmp.e2AQtfgoiD +++ rm /tmp/tmp.jo4TG2iv2w /tmp/tmp.e2AQtfgoiD +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + proxy=smart-update-haproxy + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h smart-update-pxc-0.smart-update-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Zqpjp8Zj2q +++ mktemp ++ local LAST_ERR=/tmp/tmp.oL4wXh1Uab ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Zqpjp8Zj2q ++ cat /tmp/tmp.oL4wXh1Uab ++ rm /tmp/tmp.Zqpjp8Zj2q /tmp/tmp.oL4wXh1Uab ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data smart-update select-1 no-proxy -P3306 + local cluster=smart-update + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password smart-update root ++ local cluster=smart-update ++ local user=root +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.nsZm8Qbt0l ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SUS0OezfjK +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.nsZm8Qbt0l +++ cat /tmp/tmp.SUS0OezfjK +++ rm /tmp/tmp.nsZm8Qbt0l /tmp/tmp.SUS0OezfjK +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ base64 --decode ++ local LAST_OUT=/tmp/tmp.2ayC8IHMtQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.FvzjMw61lG ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2ayC8IHMtQ ++ cat /tmp/tmp.FvzjMw61lG ++ rm /tmp/tmp.2ayC8IHMtQ /tmp/tmp.FvzjMw61lG ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size smart-update ++ local cluster=smart-update ++ local size=0 +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dgKMWu31ct ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BlEyt2cE8B +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.dgKMWu31ct +++ cat /tmp/tmp.BlEyt2cE8B +++ rm /tmp/tmp.dgKMWu31ct /tmp/tmp.BlEyt2cE8B +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UpF8pHk4Kz +++ mktemp ++ local LAST_ERR=/tmp/tmp.RgTi4L4drG ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UpF8pHk4Kz ++ cat /tmp/tmp.RgTi4L4drG ++ rm /tmp/tmp.UpF8pHk4Kz /tmp/tmp.RgTi4L4drG ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rkimjoTODx +++ mktemp ++ local LAST_ERR=/tmp/tmp.ed5x6vNb49 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rkimjoTODx ++ cat /tmp/tmp.ed5x6vNb49 ++ rm /tmp/tmp.rkimjoTODx /tmp/tmp.ed5x6vNb49 ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NNCUJkG0Vl +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ac13G4xQ2H ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NNCUJkG0Vl ++ cat /tmp/tmp.Ac13G4xQ2H ++ rm /tmp/tmp.NNCUJkG0Vl /tmp/tmp.Ac13G4xQ2H ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + grep -E -o 'early-plugin-load=keyring_\w+.so' + kubectl exec smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + return 1 ++ grep :8.0 ++ sort -V ++ tail -n2 ++ head -n1 ++ jq -r '.versions[].matrix.pxc[].imagePath' +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++++ grep -c percona-xtradb-cluster-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ head -1 +++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3gXZFSRqQO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sbMduVBM51 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.3gXZFSRqQO +++ cat /tmp/tmp.sbMduVBM51 +++ rm /tmp/tmp.3gXZFSRqQO /tmp/tmp.sbMduVBM51 +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-564f9bc786-kwmkp -n pxc-operator -- curl -s http://version-service.smart-update2-13972.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.CiV6oa0FaC +++ mktemp ++ local LAST_ERR=/tmp/tmp.DB5iHGw9Bl ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-564f9bc786-kwmkp -n pxc-operator -- curl -s http://version-service.smart-update2-13972.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CiV6oa0FaC ++ cat /tmp/tmp.DB5iHGw9Bl Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.CiV6oa0FaC /tmp/tmp.DB5iHGw9Bl ++ return 0 + vs_image=percona/percona-xtradb-cluster:8.0.41-32.1 ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CYXQZx5RNm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.W7GfhTyOfF +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.CYXQZx5RNm +++ cat /tmp/tmp.W7GfhTyOfF +++ rm /tmp/tmp.CYXQZx5RNm /tmp/tmp.W7GfhTyOfF +++ return 0 ++ client_pod=pxc-client-6df8986959-zblv5 ++ wait_pod pxc-client-6df8986959-zblv5 ++ local pod=pxc-client-6df8986959-zblv5 ++ local max_retry=480 ++ local ns= +++ grep -E '^(pxc|proxysql)$' +++ echo pxc-client-6df8986959-zblv5 +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ local container= ++ set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.41-32.1","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.8OrOoiVeuO ++ mktemp + local LAST_ERR=/tmp/tmp.Jd2ZokKlFr + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.41-32.1","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8OrOoiVeuO perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.Jd2ZokKlFr + rm /tmp/tmp.8OrOoiVeuO /tmp/tmp.Jd2ZokKlFr + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.41-32.1 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.41-32.1 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WBTsIiSnd4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.d5xopVaZgE ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WBTsIiSnd4 ++ cat /tmp/tmp.d5xopVaZgE ++ rm /tmp/tmp.WBTsIiSnd4 /tmp/tmp.d5xopVaZgE ++ return 0 + state=ready ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7gYJzFOBDC +++ mktemp ++ local LAST_ERR=/tmp/tmp.9a6lKmEcKG ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7gYJzFOBDC ++ cat /tmp/tmp.9a6lKmEcKG ++ rm /tmp/tmp.7gYJzFOBDC /tmp/tmp.9a6lKmEcKG ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.z0XPoqmxx7 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.U5TNBjVX0D +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.z0XPoqmxx7 +++++ cat /tmp/tmp.U5TNBjVX0D +++++ rm /tmp/tmp.z0XPoqmxx7 /tmp/tmp.U5TNBjVX0D +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UQv7bY7qYo +++ mktemp ++ local LAST_ERR=/tmp/tmp.It1kbP4W0h ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UQv7bY7qYo ++ cat /tmp/tmp.It1kbP4W0h ++ rm /tmp/tmp.UQv7bY7qYo /tmp/tmp.It1kbP4W0h ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return ++ seq 0 2 + for i in $(seq 0 $((CLUSTER_SIZE - 1))) + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KBfOXe3cG1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RCJrzc2WAd ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KBfOXe3cG1 ++ cat /tmp/tmp.RCJrzc2WAd ++ rm /tmp/tmp.KBfOXe3cG1 /tmp/tmp.RCJrzc2WAd ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + for i in $(seq 0 $((CLUSTER_SIZE - 1))) + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2HWMNtSIEJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.I8sbNcox3Z ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2HWMNtSIEJ ++ cat /tmp/tmp.I8sbNcox3Z ++ rm /tmp/tmp.2HWMNtSIEJ /tmp/tmp.I8sbNcox3Z ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + for i in $(seq 0 $((CLUSTER_SIZE - 1))) + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PwppCAE5NU +++ mktemp ++ local LAST_ERR=/tmp/tmp.BMu4beCppz ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PwppCAE5NU ++ cat /tmp/tmp.BMu4beCppz ++ rm /tmp/tmp.PwppCAE5NU /tmp/tmp.BMu4beCppz ++ return 0 + client_pod=pxc-client-6df8986959-zblv5 + wait_pod pxc-client-6df8986959-zblv5 + local pod=pxc-client-6df8986959-zblv5 + local max_retry=480 + local ns= ++ echo pxc-client-6df8986959-zblv5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6df8986959-zblv5 condition met waiting for pod/pxc-client-6df8986959-zblv5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.WkA2TS2tBT/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.WkA2TS2tBT/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.WkA2TS2tBT/select-1.sql + kubectl_bin delete -f /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Y1UQ7yxgFR ++ mktemp + local LAST_ERR=/tmp/tmp.mJNJJM2Hwv + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /tmp/tmp.WkA2TS2tBT/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Y1UQ7yxgFR perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-13972 namespace + cat /tmp/tmp.mJNJJM2Hwv + rm /tmp/tmp.Y1UQ7yxgFR /tmp/tmp.mJNJJM2Hwv + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.JQpLkrpEsH ++ mktemp + local LAST_ERR=/tmp/tmp.GZEqOR5aw8 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JQpLkrpEsH persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-13972 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-13972 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-13972 namespace + cat /tmp/tmp.GZEqOR5aw8 + rm /tmp/tmp.JQpLkrpEsH /tmp/tmp.GZEqOR5aw8 + return 0 + desc cleanup + set +o xtrace ----------------------------------------------------------------------------------- cleanup ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.nsbD3EUQnI ++ mktemp + local LAST_ERR=/tmp/tmp.X15VwgRJyS + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nsbD3EUQnI deployment.apps "version-service" deleted from smart-update2-13972 namespace service "version-service" deleted from smart-update2-13972 namespace + cat /tmp/tmp.X15VwgRJyS + rm /tmp/tmp.nsbD3EUQnI /tmp/tmp.X15VwgRJyS + return 0 + destroy smart-update2-13972 + local namespace=smart-update2-13972 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v level=info ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ + grep -v 'get backup status: Job.batch' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' +++ grep -c percona-xtradb-cluster-operator + sort -u + grep -v 'the object has been modified' +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator + tee /tmp/tmp.WkA2TS2tBT/operator.log ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ local LAST_OUT=/tmp/tmp.SC4Y8jNcIg +++ mktemp ++ local LAST_ERR=/tmp/tmp.AVFt1W2K4f ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SC4Y8jNcIg ++ cat /tmp/tmp.AVFt1W2K4f ++ rm /tmp/tmp.SC4Y8jNcIg /tmp/tmp.AVFt1W2K4f ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-564f9bc786-kwmkp ++ mktemp + local LAST_OUT=/tmp/tmp.2olDfrTEuY ++ mktemp + local LAST_ERR=/tmp/tmp.Pqauo4Md82 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-564f9bc786-kwmkp + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2olDfrTEuY + cat /tmp/tmp.Pqauo4Md82 + rm /tmp/tmp.2olDfrTEuY /tmp/tmp.Pqauo4Md82 + return 0 /go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136 /go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:272 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:509 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:215 2026-10-05T14:45:55.345Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-10-05T14:45:55.345Z INFO setup Manager starting up {"gitCommit": "caf25420ee3d0d26f55c22c9a7661405b27337d1", "gitBranch": "PR-2546-caf25420", "buildTime": "2026-10-05T11:36:48Z", "goVersion": "go1.26.8", "os": "linux", "arch": "amd64"} 2026-10-05T14:45:55.345Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.34.9-gke.1655001"} 2026-10-05T14:45:55.349Z INFO setup Registering Components. 2026-10-05T14:45:55.382Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-10-05T14:45:55.382Z INFO controller-runtime.metrics Starting metrics server 2026-10-05T14:45:55.382Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-10-05T14:45:55.382Z INFO setup Starting the Cmd. 2026-10-05T14:45:55.382Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-10-05T14:45:55.383Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-10-05T14:45:55.383Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-10-05T14:45:55.383Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-10-05T14:45:55.383Z INFO controller-runtime.webhook Starting webhook server 2026-10-05T14:45:55.483Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-10-05T14:46:16.002Z DEBUG events percona-xtradb-cluster-operator-564f9bc786-kwmkp_17e51d13-e014-4ae2-942c-6fdd67e3ff7a became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"125ebf2b-7bb8-43a0-a34c-43b0c723e3f0","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1791211575995407009"}, "reason": "LeaderElection"} 2026-10-05T14:46:16.002Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-10-05T14:46:16.003Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-10-05T14:46:16.004Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-10-05T14:46:16.004Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-10-05T14:46:16.004Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-10-05T14:46:16.105Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-10-05T14:46:16.105Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-10-05T14:46:16.205Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-10-05T14:46:16.205Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-10-05T14:46:16.205Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-10-05T14:46:16.205Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-10-05T14:46:21.779Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"minimal-cluster","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "minimal-cluster", "reconcileID": "cd6386db-e0a0-4cea-b063-59b5264d5624", "error": "PerconaXtraDBCluster.pxc.percona.com \"minimal-cluster\" not found"} 2026-10-05T14:47:09.002Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "06271b57-2167-4a8c-9eb8-eecca282466e", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-10-05T14:47:09.020Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "06271b57-2167-4a8c-9eb8-eecca282466e", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-10-05T14:47:09.576Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "06271b57-2167-4a8c-9eb8-eecca282466e", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-10-05T14:47:09.683Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "5dd29b43-473d-4a3b-9b22-d4f3737cc638", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T14:47:09.733Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "5dd29b43-473d-4a3b-9b22-d4f3737cc638", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T14:47:09.790Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "5dd29b43-473d-4a3b-9b22-d4f3737cc638", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:47:09.845Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "5dd29b43-473d-4a3b-9b22-d4f3737cc638", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:47:09.930Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "5dd29b43-473d-4a3b-9b22-d4f3737cc638", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:47:10.009Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "5dd29b43-473d-4a3b-9b22-d4f3737cc638", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:47:11.029Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "06b8bd48-13ce-40a3-9ace-6aaa7c270389", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T14:47:11.061Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "06b8bd48-13ce-40a3-9ace-6aaa7c270389", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T14:47:57.405Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8e4edc08-54d0-4efd-b395-9ce02172f698", "user": "operator"} 2026-10-05T14:47:57.433Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8e4edc08-54d0-4efd-b395-9ce02172f698", "user": "monitor"} 2026-10-05T14:47:57.479Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8e4edc08-54d0-4efd-b395-9ce02172f698"} 2026-10-05T14:47:57.508Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8e4edc08-54d0-4efd-b395-9ce02172f698"} 2026-10-05T14:47:57.537Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8e4edc08-54d0-4efd-b395-9ce02172f698", "user": "xtrabackup"} 2026-10-05T14:47:57.575Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8e4edc08-54d0-4efd-b395-9ce02172f698"} 2026-10-05T14:47:57.601Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8e4edc08-54d0-4efd-b395-9ce02172f698", "user": "replication"} 2026-10-05T14:47:59.609Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8e4edc08-54d0-4efd-b395-9ce02172f698", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.236.159:3306: connect: connection refused"} 2026-10-05T14:50:39.913Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "bf7b78e9-0ed2-4725-9ea5-4e3bfee44726", "user": "root"} 2026-10-05T14:50:41.171Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "bf7b78e9-0ed2-4725-9ea5-4e3bfee44726", "new version": "8.0.41-32.1"} 2026-10-05T14:54:52.555Z INFO add new job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "c2b592f5-8a97-441b-934d-7877e7e5d9fd", "name": "ensure-version/smart-update2-13972/smart-update", "schedule": "* * * * *"} 2026-10-05T14:55:52.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "c2b592f5-8a97-441b-934d-7877e7e5d9fd", "endpoint": "http://version-service.smart-update2-13972.svc.cluster.local:11000"} 2026-10-05T14:55:52.062Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "c2b592f5-8a97-441b-934d-7877e7e5d9fd", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=8ba5f13b-1ee1-400e-9beb-bdacdb935f46&databaseVersion=8.0.41-32.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=8ba5f13b-1ee1-400e-9beb-bdacdb935f46&databaseVersion=8.0.41-32.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-10-05T14:56:52.001Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "c2b592f5-8a97-441b-934d-7877e7e5d9fd", "endpoint": "http://version-service.smart-update2-13972.svc.cluster.local:11000"} 2026-10-05T14:56:52.039Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "c2b592f5-8a97-441b-934d-7877e7e5d9fd", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=8ba5f13b-1ee1-400e-9beb-bdacdb935f46&databaseVersion=8.0.41-32.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=8ba5f13b-1ee1-400e-9beb-bdacdb935f46&databaseVersion=8.0.41-32.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-10-05T14:57:31.994Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "469eb8c7-f543-413b-80da-ac5bf4dd2a0c", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2026-10-05T14:57:51.573Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "7e0ba3f7-37b8-4891-9bf9-f2e350bdf93f", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-10-05T14:57:51.590Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "7e0ba3f7-37b8-4891-9bf9-f2e350bdf93f", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-10-05T14:57:52.146Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "7e0ba3f7-37b8-4891-9bf9-f2e350bdf93f", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-10-05T14:57:52.147Z INFO cluster is not ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "c2b592f5-8a97-441b-934d-7877e7e5d9fd"} 2026-10-05T14:57:52.255Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8a26158c-623d-4685-a698-725fffb5292d", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T14:57:52.303Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8a26158c-623d-4685-a698-725fffb5292d", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T14:57:52.362Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8a26158c-623d-4685-a698-725fffb5292d", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:57:52.433Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8a26158c-623d-4685-a698-725fffb5292d", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:57:52.482Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8a26158c-623d-4685-a698-725fffb5292d", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:57:52.612Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "8a26158c-623d-4685-a698-725fffb5292d", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:57:53.450Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "fcd952a2-6d3e-4fd0-bad9-1c5b43026fa7", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T14:57:53.469Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "fcd952a2-6d3e-4fd0-bad9-1c5b43026fa7", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T15:01:28.989Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "81bea920-efaa-40df-ab04-59f36e55e393", "new version": "8.0.41-32.1"} 2026-10-05T15:04:53.469Z INFO add new job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "adc6430b-89f1-4fdc-a687-f79c2c2016e2", "name": "ensure-version/smart-update2-13972/smart-update", "schedule": "* * * * *"} 2026-10-05T15:05:53.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "adc6430b-89f1-4fdc-a687-f79c2c2016e2", "endpoint": "http://version-service.smart-update2-13972.svc.cluster.local:11000"} 2026-10-05T15:05:53.493Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "adc6430b-89f1-4fdc-a687-f79c2c2016e2", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=c0ca80cd-30fe-431d-a194-d7a26dad1c3a&databaseVersion=8.0.41-32.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=c0ca80cd-30fe-431d-a194-d7a26dad1c3a&databaseVersion=8.0.41-32.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-10-05T15:06:53.001Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "adc6430b-89f1-4fdc-a687-f79c2c2016e2", "endpoint": "http://version-service.smart-update2-13972.svc.cluster.local:11000"} 2026-10-05T15:06:53.051Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "adc6430b-89f1-4fdc-a687-f79c2c2016e2", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=c0ca80cd-30fe-431d-a194-d7a26dad1c3a&databaseVersion=8.0.41-32.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=c0ca80cd-30fe-431d-a194-d7a26dad1c3a&databaseVersion=8.0.41-32.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-10-05T15:07:46.525Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "942d6879-5927-4fa8-8c98-5d5fa56d009b", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2026-10-05T15:07:53.002Z INFO cluster is not found, deleting the job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "adc6430b-89f1-4fdc-a687-f79c2c2016e2", "name": "ensure-version/smart-update2-13972/smart-update", "cluster": "smart-update", "namespace": "smart-update2-13972"} 2026-10-05T15:08:08.898Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "809883fc-1dae-401e-a1b4-5f00a9b46a12", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-10-05T15:08:08.913Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "809883fc-1dae-401e-a1b4-5f00a9b46a12", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-10-05T15:08:09.474Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "809883fc-1dae-401e-a1b4-5f00a9b46a12", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-10-05T15:08:09.579Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "29979efb-f69f-4b28-8e67-fecda166973c", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T15:08:09.638Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "29979efb-f69f-4b28-8e67-fecda166973c", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T15:08:09.697Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "29979efb-f69f-4b28-8e67-fecda166973c", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T15:08:09.765Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "29979efb-f69f-4b28-8e67-fecda166973c", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T15:08:09.822Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "29979efb-f69f-4b28-8e67-fecda166973c", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T15:08:09.976Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "29979efb-f69f-4b28-8e67-fecda166973c", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T15:08:10.945Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "c2541a96-5e96-4cd4-9ca9-4c17da540be9", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T15:08:11.059Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "c2541a96-5e96-4cd4-9ca9-4c17da540be9", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T15:09:35.271Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "290fbdbe-8de7-41fb-985f-b28dab348818", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.232.7:3306: connect: connection refused"} 2026-10-05T15:12:04.297Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "d8ef44c0-ec88-4552-9a19-721db93602bd", "new version": "8.0.41-32.1"} 2026-10-05T15:14:23.748Z INFO add new job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "3a7d90b4-24f7-4f22-97c4-16eb143732b8", "name": "ensure-version/smart-update2-13972/smart-update", "schedule": "* * * * *"} 2026-10-05T15:15:23.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "3a7d90b4-24f7-4f22-97c4-16eb143732b8", "endpoint": "http://version-service.smart-update2-13972.svc.cluster.local:11000"} 2026-10-05T15:15:23.474Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "3a7d90b4-24f7-4f22-97c4-16eb143732b8", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.41-32.1?clusterWideEnabled=true&customResourceUid=f3d5152d-84da-4e3a-8cef-ec61139a8be5&databaseVersion=8.0.41-32.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.41-32.1?clusterWideEnabled=true&customResourceUid=f3d5152d-84da-4e3a-8cef-ec61139a8be5&databaseVersion=8.0.41-32.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-10-05T15:16:24.898Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "25a49d93-bcb0-4b2d-a262-9d7cd054b38e", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2026-10-05T15:16:24.898Z INFO cluster is not found, deleting the job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-13972"}, "namespace": "smart-update2-13972", "name": "smart-update", "reconcileID": "3a7d90b4-24f7-4f22-97c4-16eb143732b8", "name": "ensure-version/smart-update2-13972/smart-update", "cluster": "smart-update", "namespace": "smart-update2-13972"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1 github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1 github.com/robfig/cron/v3.(*Cron).startJob.func1 github.com/robfig/cron/v3.FuncJob.Run sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler + local backup_names ++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + backup_names= + [[ -n '' ]] + local pxc_names ++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + pxc_names= + [[ -n '' ]] + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.zpLO5JjNIC ++ mktemp + local LAST_ERR=/tmp/tmp.fC3Lw4eu3Q + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zpLO5JjNIC validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.fC3Lw4eu3Q + rm /tmp/tmp.zpLO5JjNIC /tmp/tmp.fC3Lw4eu3Q + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml namespace "cert-manager" deleted customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted serviceaccount "cert-manager-cainjector" deleted from cert-manager namespace serviceaccount "cert-manager" deleted from cert-manager namespace serviceaccount "cert-manager-webhook" deleted from cert-manager namespace clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace role.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace service "cert-manager-cainjector" deleted from cert-manager namespace service "cert-manager" deleted from cert-manager namespace service "cert-manager-webhook" deleted from cert-manager namespace deployment.apps "cert-manager-cainjector" deleted from cert-manager namespace deployment.apps "cert-manager" deleted from cert-manager namespace deployment.apps "cert-manager-webhook" deleted from cert-manager namespace mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace smart-update2-13972 + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ date +%s ++ mktemp + local LAST_OUT=/tmp/tmp.QKpYTKz12K + local deadline=1791213548 + local LAST_OUT=/tmp/tmp.9LAHnSm8GK ++ mktemp ++ kubectl_bin get namespace smart-update2-13972 --ignore-not-found -o name ++ mktemp + local LAST_ERR=/tmp/tmp.wl3RlxYuSc + local exit_status=0 + local LAST_ERR=/tmp/tmp.DA3W6HHQkb + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace smart-update2-13972 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + [[ -z namespace/smart-update2-13972 ]] ++ date +%s + [[ 1791213430 -ge 1791213548 ]] + sleep 5 ++ kubectl_bin get namespace smart-update2-13972 --ignore-not-found -o name + [[ -z namespace/smart-update2-13972 ]] ++ date +%s + [[ 1791213436 -ge 1791213548 ]] + sleep 5 ++ kubectl_bin get namespace smart-update2-13972 --ignore-not-found -o name + [[ -z namespace/smart-update2-13972 ]] ++ date +%s + [[ 1791213442 -ge 1791213548 ]] + sleep 5 ++ kubectl_bin get namespace smart-update2-13972 --ignore-not-found -o name + [[ -z namespace/smart-update2-13972 ]] ++ date +%s + [[ 1791213448 -ge 1791213548 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9LAHnSm8GK namespace "pxc-operator" force deleted + cat /tmp/tmp.DA3W6HHQkb Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.9LAHnSm8GK /tmp/tmp.DA3W6HHQkb + return 0 ++ kubectl_bin get namespace smart-update2-13972 --ignore-not-found -o name + [[ -z namespace/smart-update2-13972 ]] ++ date +%s + [[ 1791213454 -ge 1791213548 ]] + sleep 5 ++ kubectl_bin get namespace smart-update2-13972 --ignore-not-found -o name + [[ -z namespace/smart-update2-13972 ]] ++ date +%s + [[ 1791213460 -ge 1791213548 ]] + sleep 5 ++ kubectl_bin get namespace smart-update2-13972 --ignore-not-found -o name + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QKpYTKz12K namespace "smart-update2-13972" force deleted + cat /tmp/tmp.wl3RlxYuSc Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.QKpYTKz12K /tmp/tmp.wl3RlxYuSc + return 0 + [[ -z '' ]] ++ date +%s + deadline=1791213586 ++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' ++ awk '$2 != ""' + [[ -z '' ]] + rm -rf /tmp/tmp.WkA2TS2tBT + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------