Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/logs/smart-update2-8-4.log Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 + API=pxc.percona.com/v9-9-9 + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + CLUSTER=smart-update + CLUSTER_SIZE=3 + PROXY_SIZE=2 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + PXC_VER=8.4 + TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + VS_URL=http://version-service + VS_PORT=11000 + VS_ENDPOINT=http://version-service:11000 + [[ 8.4 == \8\.\4 ]] + VS_UPDATE_STRATEGY=latest + main + create_infra smart-update2-14904 + local ns=smart-update2-14904 + '[' -n pxc-operator ']' + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + kubectl patch pxc -n smart-update2-23819 minimal-cluster --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/minimal-cluster patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.YxSejDZa8R ++ mktemp + local LAST_ERR=/tmp/tmp.jUMOrZTLTu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YxSejDZa8R perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-23819 namespace + cat /tmp/tmp.jUMOrZTLTu + rm /tmp/tmp.YxSejDZa8R /tmp/tmp.jUMOrZTLTu + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.ejPcrqBlWm ++ mktemp + local LAST_ERR=/tmp/tmp.IR452DwGN9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ejPcrqBlWm No resources found + cat /tmp/tmp.IR452DwGN9 + rm /tmp/tmp.ejPcrqBlWm /tmp/tmp.IR452DwGN9 + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.7i9ZQCfkPP ++ mktemp + local LAST_ERR=/tmp/tmp.zNxEYhpbZP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7i9ZQCfkPP No resources found + cat /tmp/tmp.zNxEYhpbZP + rm /tmp/tmp.7i9ZQCfkPP /tmp/tmp.zNxEYhpbZP + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + kubectl_bin get ns + xargs kubectl delete ns + awk '{print$1}' ++ mktemp + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + local LAST_OUT=/tmp/tmp.kleEY1R1GT ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.Q0qwIxsv19 + local LAST_ERR=/tmp/tmp.2IWqOOhhS1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator ++ mktemp + local LAST_ERR=/tmp/tmp.8TlfzOYOOG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Q0qwIxsv19 + cat /tmp/tmp.8TlfzOYOOG + rm /tmp/tmp.Q0qwIxsv19 /tmp/tmp.8TlfzOYOOG + return 0 namespace "cert-manager" deleted namespace "smart-update2-23819" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kleEY1R1GT namespace "pxc-operator" deleted + cat /tmp/tmp.2IWqOOhhS1 + rm /tmp/tmp.kleEY1R1GT /tmp/tmp.2IWqOOhhS1 + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.qenaqIQrVs ++ mktemp + local LAST_ERR=/tmp/tmp.Pn6CtXAe38 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qenaqIQrVs namespace/pxc-operator created + cat /tmp/tmp.Pn6CtXAe38 + rm /tmp/tmp.qenaqIQrVs /tmp/tmp.Pn6CtXAe38 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.nQLWDwMLnP +++ mktemp ++ local LAST_ERR=/tmp/tmp.6wF8YYzZIr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nQLWDwMLnP ++ cat /tmp/tmp.6wF8YYzZIr ++ rm /tmp/tmp.nQLWDwMLnP /tmp/tmp.6wF8YYzZIr ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster1 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.CXBrbqOEfp ++ mktemp + local LAST_ERR=/tmp/tmp.C3u7Ub5Ona + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster1 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CXBrbqOEfp Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster1" modified. + cat /tmp/tmp.C3u7Ub5Ona + rm /tmp/tmp.CXBrbqOEfp /tmp/tmp.C3u7Ub5Ona + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.ACC6U0pUEy ++ mktemp + local LAST_ERR=/tmp/tmp.KxceDDEg9O + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ACC6U0pUEy customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.KxceDDEg9O + rm /tmp/tmp.ACC6U0pUEy /tmp/tmp.KxceDDEg9O + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.m01n0KFagS ++ mktemp + local LAST_ERR=/tmp/tmp.fTuPf2cUas + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.m01n0KFagS clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.fTuPf2cUas + rm /tmp/tmp.m01n0KFagS /tmp/tmp.fTuPf2cUas + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/cw-operator.yaml + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AwEQooYyfH ++ mktemp + local LAST_ERR=/tmp/tmp.OVDOvycXXk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AwEQooYyfH deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.OVDOvycXXk + rm /tmp/tmp.AwEQooYyfH /tmp/tmp.OVDOvycXXk + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.HhY5ljai5E ++ mktemp + local LAST_ERR=/tmp/tmp.Psa7NpKe6i + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HhY5ljai5E pod/percona-xtradb-cluster-operator-665ff7485-bp5ln condition met + cat /tmp/tmp.Psa7NpKe6i + rm /tmp/tmp.HhY5ljai5E /tmp/tmp.Psa7NpKe6i + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.EHkU14RlYv +++ mktemp ++ local LAST_ERR=/tmp/tmp.MWzni5winu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EHkU14RlYv ++ cat /tmp/tmp.MWzni5winu ++ rm /tmp/tmp.EHkU14RlYv /tmp/tmp.MWzni5winu ++ return 0 + wait_pod percona-xtradb-cluster-operator-665ff7485-bp5ln 480 pxc-operator + local pod=percona-xtradb-cluster-operator-665ff7485-bp5ln + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-665ff7485-bp5ln ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-665ff7485-bp5ln condition met waiting for pod/percona-xtradb-cluster-operator-665ff7485-bp5ln to become Ready.Ok + sleep 3 + create_namespace smart-update2-14904 + local namespace=smart-update2-14904 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrole ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces smart-update2-14904' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces smart-update2-14904 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace smart-update2-14904 + xargs kubectl delete ns + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + kubectl_bin get ns + local LAST_OUT=/tmp/tmp.KkPq8wuMZ3 ++ mktemp + local LAST_ERR=/tmp/tmp.ZBzTjrcf30 + local exit_status=0 ++ mktemp + local LAST_OUT=/tmp/tmp.wJDCZd9Umv ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-14904 ++ mktemp + local LAST_ERR=/tmp/tmp.7cnapbyiA2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wJDCZd9Umv + cat /tmp/tmp.7cnapbyiA2 + rm /tmp/tmp.wJDCZd9Umv /tmp/tmp.7cnapbyiA2 + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-14904 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-14904 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.KkPq8wuMZ3 + cat /tmp/tmp.ZBzTjrcf30 Error from server (NotFound): namespaces "smart-update2-14904" not found + rm /tmp/tmp.KkPq8wuMZ3 /tmp/tmp.ZBzTjrcf30 + return 1 + : + wait_for_delete namespace/smart-update2-14904 + local res=namespace/smart-update2-14904 + echo -n 'waiting for namespace/smart-update2-14904 to be deleted' waiting for namespace/smart-update2-14904 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "smart-update2-14904" not found + desc 'create namespace smart-update2-14904' + set +o xtrace ----------------------------------------------------------------------------------- create namespace smart-update2-14904 ----------------------------------------------------------------------------------- + kubectl_bin create namespace smart-update2-14904 ++ mktemp + local LAST_OUT=/tmp/tmp.JTV2IQyDgx ++ mktemp + local LAST_ERR=/tmp/tmp.Pcz6X7Rhac + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace smart-update2-14904 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JTV2IQyDgx namespace/smart-update2-14904 created + cat /tmp/tmp.Pcz6X7Rhac + rm /tmp/tmp.JTV2IQyDgx /tmp/tmp.Pcz6X7Rhac + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.u9hp0g8MXJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.GRvk1dWnoG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u9hp0g8MXJ ++ cat /tmp/tmp.GRvk1dWnoG ++ rm /tmp/tmp.u9hp0g8MXJ /tmp/tmp.GRvk1dWnoG ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster1 --namespace=smart-update2-14904 ++ mktemp + local LAST_OUT=/tmp/tmp.vnjUMLG7Fl ++ mktemp + local LAST_ERR=/tmp/tmp.WN2Uu7Fhb5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster1 --namespace=smart-update2-14904 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vnjUMLG7Fl Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster1" modified. + cat /tmp/tmp.WN2Uu7Fhb5 + rm /tmp/tmp.vnjUMLG7Fl /tmp/tmp.WN2Uu7Fhb5 + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.8CnnFRG4vM ++ mktemp + local LAST_ERR=/tmp/tmp.lhLRP7MGxX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8CnnFRG4vM secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.lhLRP7MGxX + rm /tmp/tmp.8CnnFRG4vM /tmp/tmp.lhLRP7MGxX + return 0 + deploy_version_service + desc 'install version service' + set +o xtrace ----------------------------------------------------------------------------------- install version service ----------------------------------------------------------------------------------- + kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json ++ mktemp + local LAST_OUT=/tmp/tmp.me3dn2214s ++ mktemp + local LAST_ERR=/tmp/tmp.yVhCxDQT5p + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.me3dn2214s configmap/versions created + cat /tmp/tmp.yVhCxDQT5p + rm /tmp/tmp.me3dn2214s /tmp/tmp.yVhCxDQT5p + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.II7oPo1Vla ++ mktemp + local LAST_ERR=/tmp/tmp.vEIyul2vFm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.II7oPo1Vla deployment.apps/version-service created service/version-service created + cat /tmp/tmp.vEIyul2vFm + rm /tmp/tmp.II7oPo1Vla /tmp/tmp.vEIyul2vFm + return 0 + sleep 10 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.iH7ujs8C7d ++ mktemp + local LAST_ERR=/tmp/tmp.tlTdzZqMzM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iH7ujs8C7d namespace/cert-manager created + cat /tmp/tmp.tlTdzZqMzM + rm /tmp/tmp.iH7ujs8C7d /tmp/tmp.tlTdzZqMzM + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.nVRCo137nR ++ mktemp + local LAST_ERR=/tmp/tmp.T1SxhkJUUO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nVRCo137nR namespace/cert-manager labeled + cat /tmp/tmp.T1SxhkJUUO + rm /tmp/tmp.nVRCo137nR /tmp/tmp.T1SxhkJUUO + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.vU58esRzZb ++ mktemp + local LAST_ERR=/tmp/tmp.BID8CVYnlT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vU58esRzZb namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.BID8CVYnlT Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.vU58esRzZb /tmp/tmp.BID8CVYnlT + return 0 + '[' '' == 4.10 ']' + sleep 70 ++ jq -r '.versions[].matrix.pxc[].imagePath' +++ get_operator_pod ++ grep :8.4 +++ local label_prefix=app.kubernetes.io/ ++ sort -V ++ head -n1 ++ tail -n3 ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++++ grep -c percona-xtradb-cluster-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++++ mktemp +++ local LAST_OUT=/tmp/tmp.elH2jES85Y +++ head -1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bSjzBb2nO2 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.elH2jES85Y +++ cat /tmp/tmp.bSjzBb2nO2 +++ rm /tmp/tmp.elH2jES85Y /tmp/tmp.bSjzBb2nO2 +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-665ff7485-bp5ln -n pxc-operator -- curl -s http://version-service.smart-update2-14904.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.tbNmHCo8mk +++ mktemp ++ local LAST_ERR=/tmp/tmp.GhhI7VGOpJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-665ff7485-bp5ln -n pxc-operator -- curl -s http://version-service.smart-update2-14904.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tbNmHCo8mk ++ cat /tmp/tmp.GhhI7VGOpJ Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.tbNmHCo8mk /tmp/tmp.GhhI7VGOpJ ++ return 0 + IMAGE_PXC=percona/percona-xtradb-cluster:8.4.5-5.1 + kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' ++ mktemp + local LAST_OUT=/tmp/tmp.W3TEWoykZU ++ mktemp + local LAST_ERR=/tmp/tmp.Nyb9ul6J8G + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.W3TEWoykZU customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched + cat /tmp/tmp.Nyb9ul6J8G + rm /tmp/tmp.W3TEWoykZU /tmp/tmp.Nyb9ul6J8G + return 0 + kubectl_bin -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-14904.svc.cluster.local:11000 ++ mktemp + local LAST_OUT=/tmp/tmp.7NLPdz8tIx ++ mktemp + local LAST_ERR=/tmp/tmp.keVdnelyUg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-14904.svc.cluster.local:11000 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7NLPdz8tIx deployment.apps/percona-xtradb-cluster-operator env updated + cat /tmp/tmp.keVdnelyUg + rm /tmp/tmp.7NLPdz8tIx /tmp/tmp.keVdnelyUg + return 0 + desc 'Starting telemetry testing' + set +o xtrace ----------------------------------------------------------------------------------- Starting telemetry testing ----------------------------------------------------------------------------------- + kubectl_bin apply -f - + /usr/bin/sed s/version-service/version-service-cr/g /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.5g6EpSzud2 ++ yq 'select(.kind == "Deployment").spec.template.spec.containers[0].image' /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_ERR=/tmp/tmp.QAc52lvvJi + local exit_status=0 ++ seq 0 2 + yq eval '(. | select(.kind == "Deployment") | .spec.template.spec.containers[0].image) = "perconalab/version-service:main-latest"' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5g6EpSzud2 deployment.apps/version-service-cr created service/version-service-cr created + cat /tmp/tmp.QAc52lvvJi + rm /tmp/tmp.5g6EpSzud2 /tmp/tmp.QAc52lvvJi + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.1imbeVtBbD ++ mktemp + local LAST_ERR=/tmp/tmp.Q6Wbr0xTsu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1imbeVtBbD pod "version-service-559c4df55c-4x6vc" deleted from smart-update2-14904 namespace + cat /tmp/tmp.Q6Wbr0xTsu + rm /tmp/tmp.1imbeVtBbD /tmp/tmp.Q6Wbr0xTsu + return 0 ++ echo -n percona/percona-xtradb-cluster:8.4.5-5.1 ++ sed -r 's/^.*:([0-9]+.[0-9]+).*/\1/' + IMAGE_PREFIX=8.4 + desc 'Enable telemetry on operator level' + set +o xtrace ----------------------------------------------------------------------------------- Enable telemetry on operator level ----------------------------------------------------------------------------------- + yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "false"' + kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + kubectl_bin apply -n pxc-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.SPuGx2mg00 ++ mktemp + local LAST_OUT=/tmp/tmp.MOUnA26XQz ++ mktemp + local LAST_ERR=/tmp/tmp.ImQXvDMZSq + local exit_status=0 ++ mktemp ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n pxc-operator -f - + local LAST_ERR=/tmp/tmp.puJnsVvdTF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SPuGx2mg00 + cat /tmp/tmp.puJnsVvdTF + rm /tmp/tmp.SPuGx2mg00 /tmp/tmp.puJnsVvdTF + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MOUnA26XQz deployment.apps/percona-xtradb-cluster-operator configured + cat /tmp/tmp.ImQXvDMZSq + rm /tmp/tmp.MOUnA26XQz /tmp/tmp.ImQXvDMZSq + return 0 + sleep 30 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.IqBby5bKdB +++ mktemp ++ local LAST_ERR=/tmp/tmp.yXOIcMwiSq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IqBby5bKdB ++ cat /tmp/tmp.yXOIcMwiSq ++ rm /tmp/tmp.IqBby5bKdB /tmp/tmp.yXOIcMwiSq ++ return 0 + wait_pod percona-xtradb-cluster-operator-5f78f4fd46-vgpjq 480 pxc-operator + local pod=percona-xtradb-cluster-operator-5f78f4fd46-vgpjq + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-5f78f4fd46-vgpjq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-5f78f4fd46-vgpjq condition met waiting for pod/percona-xtradb-cluster-operator-5f78f4fd46-vgpjq to become Ready.Ok + check_telemetry_transfer http://version-service-cr.smart-update2-14904.svc.cluster.local:11000 disabled enabled + local cr_vs_uri=http://version-service-cr.smart-update2-14904.svc.cluster.local:11000 + local cr_vs_channel=disabled + local telemetry_state=enabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.eCxt24NnAF ++ mktemp + local LAST_ERR=/tmp/tmp.LwaLoY5JqW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eCxt24NnAF deployment.apps/pxc-client created + cat /tmp/tmp.LwaLoY5JqW + rm /tmp/tmp.eCxt24NnAF /tmp/tmp.LwaLoY5JqW + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.VFYNHTc0EH ++ mktemp + local LAST_ERR=/tmp/tmp.fNHcRX6WBK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VFYNHTc0EH secret/minimal-cluster created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.fNHcRX6WBK + rm /tmp/tmp.VFYNHTc0EH /tmp/tmp.fNHcRX6WBK + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/cr-minimal.yaml + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-14904.svc.cluster.local:11000"' + yq eval '.spec.upgradeOptions.apply="disabled"' + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.4.5-5.1"' + yq eval '.spec.crVersion="9.9.9"' + yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"' + kubectl_bin apply -f - + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe"' ++ mktemp + local LAST_OUT=/tmp/tmp.Up5F7OCh8J ++ mktemp + local LAST_ERR=/tmp/tmp.9GXGJtuen3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Up5F7OCh8J perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.9GXGJtuen3 + rm /tmp/tmp.Up5F7OCh8J /tmp/tmp.9GXGJtuen3 + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zsgjLeafU2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Jf9U156a21 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.zsgjLeafU2 +++ cat /tmp/tmp.Jf9U156a21 +++ rm /tmp/tmp.zsgjLeafU2 /tmp/tmp.Jf9U156a21 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FMiDH6vBvM +++ mktemp ++ local LAST_ERR=/tmp/tmp.feI1YVCxtb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FMiDH6vBvM ++ cat /tmp/tmp.feI1YVCxtb ++ rm /tmp/tmp.FMiDH6vBvM /tmp/tmp.feI1YVCxtb ++ return 0 + client_pod=pxc-client-67fc4995bb-lbkvd + wait_pod pxc-client-67fc4995bb-lbkvd + local pod=pxc-client-67fc4995bb-lbkvd + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-lbkvd ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-lbkvd condition met waiting for pod/pxc-client-67fc4995bb-lbkvd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sc0XwIQSZV +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZtPo2oYwvj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sc0XwIQSZV ++ cat /tmp/tmp.ZtPo2oYwvj ++ rm /tmp/tmp.sc0XwIQSZV /tmp/tmp.ZtPo2oYwvj ++ return 0 + client_pod=pxc-client-67fc4995bb-lbkvd + wait_pod pxc-client-67fc4995bb-lbkvd + local pod=pxc-client-67fc4995bb-lbkvd + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-lbkvd ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-lbkvd condition met waiting for pod/pxc-client-67fc4995bb-lbkvd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + grep -E 'server request payload|unary call' + grep -Eo '\{.*\}' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.time_ms")' + kubectl_bin logs version-service-cr-76d4659869-hq6hx ++ mktemp + local LAST_OUT=/tmp/tmp.qfERybEtS9 ++ mktemp + local LAST_ERR=/tmp/tmp.WJXSr6XiJG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-76d4659869-hq6hx + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qfERybEtS9 + cat /tmp/tmp.WJXSr6XiJG + rm /tmp/tmp.qfERybEtS9 /tmp/tmp.WJXSr6XiJG + return 0 + grep -E 'server request payload|unary call' + grep -Eo '\{.*\}' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.time_ms")' + kubectl_bin logs version-service-559c4df55c-kq5ss ++ mktemp + local LAST_OUT=/tmp/tmp.k1S52fOJ0S ++ mktemp + local LAST_ERR=/tmp/tmp.VYQYLEGdCo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-559c4df55c-kq5ss + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.k1S52fOJ0S + cat /tmp/tmp.VYQYLEGdCo + rm /tmp/tmp.k1S52fOJ0S /tmp/tmp.VYQYLEGdCo + return 0 + local telemetry_log_file=enabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' disabled == disabled -a enabled == enabled ']' + desc 'operator fallback VS should have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator fallback VS should have telemetry ----------------------------------------------------------------------------------- + diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json /dev/fd/63 ++ grep -f /tmp/tmp.eYNH1l0X9V/enabled_telemetry.version-service.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json + desc 'CR VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- CR VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.eYNH1l0X9V/enabled_telemetry.version-service-cr.log.json ]] + local image_prefix=disabled + image_prefix=disabled + local telemetry_cr_log_file=enabled_telemetry.version-service-cr-disabled-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + [[ disabled == \d\i\s\a\b\l\e\d\-\r\e\c\o\m\m\e\n\d\e\d ]] + [[ disabled == \d\i\s\a\b\l\e\d\-\l\a\t\e\s\t ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' disabled == disabled -a enabled == disabled ']' + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.2mTSCXxWFI ++ mktemp + local LAST_ERR=/tmp/tmp.GeObwdVH5Y + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2mTSCXxWFI perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.GeObwdVH5Y + rm /tmp/tmp.2mTSCXxWFI /tmp/tmp.GeObwdVH5Y + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.0lTd1iciIg +++ mktemp ++ local LAST_ERR=/tmp/tmp.Jn9gKuB2HG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0lTd1iciIg ++ cat /tmp/tmp.Jn9gKuB2HG ++ rm /tmp/tmp.0lTd1iciIg /tmp/tmp.Jn9gKuB2HG ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-5f78f4fd46-vgpjq ++ mktemp + local LAST_OUT=/tmp/tmp.RvRAsMHT02 ++ mktemp + local LAST_ERR=/tmp/tmp.KNX3lCeY8C + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-5f78f4fd46-vgpjq + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RvRAsMHT02 pod "percona-xtradb-cluster-operator-5f78f4fd46-vgpjq" deleted from pxc-operator namespace + cat /tmp/tmp.KNX3lCeY8C + rm /tmp/tmp.RvRAsMHT02 /tmp/tmp.KNX3lCeY8C + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.uV1pZ7M9Hh ++ mktemp + local LAST_ERR=/tmp/tmp.tyXEzEWj33 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uV1pZ7M9Hh perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-14904 namespace + cat /tmp/tmp.tyXEzEWj33 + rm /tmp/tmp.uV1pZ7M9Hh /tmp/tmp.tyXEzEWj33 + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.gcSfTskka7 ++ mktemp + local LAST_ERR=/tmp/tmp.sfhGuCQeoo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gcSfTskka7 deployment.apps "pxc-client" deleted from smart-update2-14904 namespace + cat /tmp/tmp.sfhGuCQeoo + rm /tmp/tmp.gcSfTskka7 /tmp/tmp.sfhGuCQeoo + return 0 + sleep 30 + desc 'Disabling telemetry on the operator level' + set +o xtrace ----------------------------------------------------------------------------------- Disabling telemetry on the operator level ----------------------------------------------------------------------------------- + kubectl_bin delete pod -l run=version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.rfH27adcKc ++ mktemp + local LAST_ERR=/tmp/tmp.ok62J1Pkyl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rfH27adcKc pod "version-service-cr-76d4659869-hq6hx" deleted from smart-update2-14904 namespace + cat /tmp/tmp.ok62J1Pkyl + rm /tmp/tmp.rfH27adcKc /tmp/tmp.ok62J1Pkyl + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.LUQuj2zeSY ++ mktemp + local LAST_ERR=/tmp/tmp.ScWTWQFgjQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LUQuj2zeSY pod "version-service-559c4df55c-kq5ss" deleted from smart-update2-14904 namespace + cat /tmp/tmp.ScWTWQFgjQ + rm /tmp/tmp.LUQuj2zeSY /tmp/tmp.ScWTWQFgjQ + return 0 + kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + kubectl_bin apply -n pxc-operator -f - ++ mktemp + yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' ++ mktemp + local LAST_OUT=/tmp/tmp.hvttqayfGd + local LAST_OUT=/tmp/tmp.TuqA2DnzUU ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.lDJEJxvMwX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + local LAST_ERR=/tmp/tmp.UKbDmtHI90 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n pxc-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TuqA2DnzUU + cat /tmp/tmp.lDJEJxvMwX + rm /tmp/tmp.TuqA2DnzUU /tmp/tmp.lDJEJxvMwX + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hvttqayfGd deployment.apps/percona-xtradb-cluster-operator configured + cat /tmp/tmp.UKbDmtHI90 + rm /tmp/tmp.hvttqayfGd /tmp/tmp.UKbDmtHI90 + return 0 + sleep 30 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.WAGTlhrLhi +++ mktemp ++ local LAST_ERR=/tmp/tmp.QVCQYXe4Qd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WAGTlhrLhi ++ cat /tmp/tmp.QVCQYXe4Qd ++ rm /tmp/tmp.WAGTlhrLhi /tmp/tmp.QVCQYXe4Qd ++ return 0 + wait_pod percona-xtradb-cluster-operator-698c684f5f-7czqt 480 pxc-operator + local pod=percona-xtradb-cluster-operator-698c684f5f-7czqt + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-698c684f5f-7czqt ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-698c684f5f-7czqt condition met waiting for pod/percona-xtradb-cluster-operator-698c684f5f-7czqt to become Ready.Ok + check_telemetry_transfer http://version-service-cr.smart-update2-14904.svc.cluster.local:11000 8.4-latest disabled + local cr_vs_uri=http://version-service-cr.smart-update2-14904.svc.cluster.local:11000 + local cr_vs_channel=8.4-latest + local telemetry_state=disabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.H5aL0ozUTk ++ mktemp + local LAST_ERR=/tmp/tmp.Tgp5Ya1RkA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.H5aL0ozUTk deployment.apps/pxc-client created + cat /tmp/tmp.Tgp5Ya1RkA + rm /tmp/tmp.H5aL0ozUTk /tmp/tmp.Tgp5Ya1RkA + return 0 + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.8kfozPgqSQ ++ mktemp + local LAST_ERR=/tmp/tmp.dGKNNYME4i + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8kfozPgqSQ secret/minimal-cluster unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.dGKNNYME4i + rm /tmp/tmp.8kfozPgqSQ /tmp/tmp.dGKNNYME4i + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/cr-minimal.yaml + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-14904.svc.cluster.local:11000"' + yq eval '.spec.upgradeOptions.apply="8.4-latest"' + yq eval '.spec.crVersion="9.9.9"' + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.4.5-5.1"' + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + kubectl_bin apply -f - + yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"' + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe"' ++ mktemp + local LAST_OUT=/tmp/tmp.3BkXwcFvFp ++ mktemp + local LAST_ERR=/tmp/tmp.gZpQsq41Kt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3BkXwcFvFp perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.gZpQsq41Kt + rm /tmp/tmp.3BkXwcFvFp /tmp/tmp.gZpQsq41Kt + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QvwUTEEDJR ++++ mktemp +++ local LAST_ERR=/tmp/tmp.akWOlRPwDy +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.QvwUTEEDJR +++ cat /tmp/tmp.akWOlRPwDy +++ rm /tmp/tmp.QvwUTEEDJR /tmp/tmp.akWOlRPwDy +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GkkJEep9Sh +++ mktemp ++ local LAST_ERR=/tmp/tmp.BQKUkcHkOI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GkkJEep9Sh ++ cat /tmp/tmp.BQKUkcHkOI ++ rm /tmp/tmp.GkkJEep9Sh /tmp/tmp.BQKUkcHkOI ++ return 0 + client_pod=pxc-client-67fc4995bb-fm2h7 + wait_pod pxc-client-67fc4995bb-fm2h7 + local pod=pxc-client-67fc4995bb-fm2h7 + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-fm2h7 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-fm2h7 condition met waiting for pod/pxc-client-67fc4995bb-fm2h7 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YDtXxA5Z2s +++ mktemp ++ local LAST_ERR=/tmp/tmp.nnMVgdhOaQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YDtXxA5Z2s ++ cat /tmp/tmp.nnMVgdhOaQ ++ rm /tmp/tmp.YDtXxA5Z2s /tmp/tmp.nnMVgdhOaQ ++ return 0 + client_pod=pxc-client-67fc4995bb-fm2h7 + wait_pod pxc-client-67fc4995bb-fm2h7 + local pod=pxc-client-67fc4995bb-fm2h7 + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-fm2h7 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-fm2h7 condition met waiting for pod/pxc-client-67fc4995bb-fm2h7 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + grep -Eo '\{.*\}' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.time_ms")' + grep -E 'server request payload|unary call' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.start_time")' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-cr-76d4659869-b6684 ++ mktemp + local LAST_OUT=/tmp/tmp.tgmwosIH7f ++ mktemp + local LAST_ERR=/tmp/tmp.EvTiAM96ER + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-76d4659869-b6684 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tgmwosIH7f + cat /tmp/tmp.EvTiAM96ER + rm /tmp/tmp.tgmwosIH7f /tmp/tmp.EvTiAM96ER + return 0 + grep -E 'server request payload|unary call' + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + grep -Eo '\{.*\}' + jq 'del(."grpc.request.content".msg.kubeVersion)' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-559c4df55c-6cb28 ++ mktemp + local LAST_OUT=/tmp/tmp.lBGYI2k1QU ++ mktemp + local LAST_ERR=/tmp/tmp.T3BAt5DtlT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-559c4df55c-6cb28 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lBGYI2k1QU + cat /tmp/tmp.T3BAt5DtlT + rm /tmp/tmp.lBGYI2k1QU /tmp/tmp.T3BAt5DtlT + return 0 + local telemetry_log_file=disabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' 8.4-latest == disabled -a disabled == enabled ']' + local image_prefix=8.4-latest + image_prefix=8.4 + local telemetry_cr_log_file=disabled_telemetry.version-service-cr-8.4-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + [[ 8.4-latest == \8\.\4\-\r\e\c\o\m\m\e\n\d\e\d ]] + [[ 8.4-latest == \8\.\4\-\l\a\t\e\s\t ]] + '[' disabled == disabled ']' + desc 'cr VS should have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- cr VS should have telemetry ----------------------------------------------------------------------------------- + diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.4-cw.log.json /dev/fd/63 ++ grep -f /tmp/tmp.eYNH1l0X9V/disabled_telemetry.version-service-cr.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.4-cw.log.json + desc 'operator VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.eYNH1l0X9V/disabled_telemetry.version-service.log.json ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' 8.4-latest == disabled -a disabled == disabled ']' + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.4yvPzY3Nlp ++ mktemp + local LAST_ERR=/tmp/tmp.DQEYLuTKyF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4yvPzY3Nlp perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.DQEYLuTKyF + rm /tmp/tmp.4yvPzY3Nlp /tmp/tmp.DQEYLuTKyF + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.UVcjAIgnx8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.SVCr3hPHWX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UVcjAIgnx8 ++ cat /tmp/tmp.SVCr3hPHWX ++ rm /tmp/tmp.UVcjAIgnx8 /tmp/tmp.SVCr3hPHWX ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-698c684f5f-7czqt ++ mktemp + local LAST_OUT=/tmp/tmp.eiJBWSWzp1 ++ mktemp + local LAST_ERR=/tmp/tmp.Ql1NtxBhZc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-698c684f5f-7czqt + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eiJBWSWzp1 pod "percona-xtradb-cluster-operator-698c684f5f-7czqt" deleted from pxc-operator namespace + cat /tmp/tmp.Ql1NtxBhZc + rm /tmp/tmp.eiJBWSWzp1 /tmp/tmp.Ql1NtxBhZc + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.iawM7s7rxN ++ mktemp + local LAST_ERR=/tmp/tmp.CmNRCdjNuZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iawM7s7rxN perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-14904 namespace + cat /tmp/tmp.CmNRCdjNuZ + rm /tmp/tmp.iawM7s7rxN /tmp/tmp.CmNRCdjNuZ + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.gbt0m7wtlO ++ mktemp + local LAST_ERR=/tmp/tmp.vOnLJPLJiJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gbt0m7wtlO deployment.apps "pxc-client" deleted from smart-update2-14904 namespace + cat /tmp/tmp.vOnLJPLJiJ + rm /tmp/tmp.gbt0m7wtlO /tmp/tmp.vOnLJPLJiJ + return 0 + sleep 30 + kubectl_bin delete pod -l run=version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.biPonAIUMd ++ mktemp + local LAST_ERR=/tmp/tmp.XYjs0l8T37 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.biPonAIUMd pod "version-service-cr-76d4659869-b6684" deleted from smart-update2-14904 namespace + cat /tmp/tmp.XYjs0l8T37 + rm /tmp/tmp.biPonAIUMd /tmp/tmp.XYjs0l8T37 + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.4ktyt6EFlh ++ mktemp + local LAST_ERR=/tmp/tmp.kw2MpzBxKm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4ktyt6EFlh pod "version-service-559c4df55c-6cb28" deleted from smart-update2-14904 namespace + cat /tmp/tmp.kw2MpzBxKm + rm /tmp/tmp.4ktyt6EFlh /tmp/tmp.kw2MpzBxKm + return 0 + check_telemetry_transfer http://version-service-cr.smart-update2-14904.svc.cluster.local:11000 disabled disabled + local cr_vs_uri=http://version-service-cr.smart-update2-14904.svc.cluster.local:11000 + local cr_vs_channel=disabled + local telemetry_state=disabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.c6nja5CMpG ++ mktemp + local LAST_ERR=/tmp/tmp.LaBGxhRbaE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.c6nja5CMpG deployment.apps/pxc-client created + cat /tmp/tmp.LaBGxhRbaE + rm /tmp/tmp.c6nja5CMpG /tmp/tmp.LaBGxhRbaE + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.XTKSmJprJw ++ mktemp + local LAST_ERR=/tmp/tmp.dpO0BYGWAA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XTKSmJprJw secret/minimal-cluster unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.dpO0BYGWAA + rm /tmp/tmp.XTKSmJprJw /tmp/tmp.dpO0BYGWAA + return 0 + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-14904.svc.cluster.local:11000"' + yq eval '.spec.upgradeOptions.apply="disabled"' + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe"' + kubectl_bin apply -f - + yq eval '.spec.crVersion="9.9.9"' + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.4.5-5.1"' ++ mktemp + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/cr-minimal.yaml + local LAST_OUT=/tmp/tmp.qgzV3vUxLn + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' ++ mktemp + local LAST_ERR=/tmp/tmp.QJ9Qtikp40 + local exit_status=0 + yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qgzV3vUxLn perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.QJ9Qtikp40 + rm /tmp/tmp.qgzV3vUxLn /tmp/tmp.QJ9Qtikp40 + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.z9f66o6HTG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OhleSGSdpm +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.z9f66o6HTG +++ cat /tmp/tmp.OhleSGSdpm +++ rm /tmp/tmp.z9f66o6HTG /tmp/tmp.OhleSGSdpm +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4pbvFJztBO +++ mktemp ++ local LAST_ERR=/tmp/tmp.sp7ciTlRyb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4pbvFJztBO ++ cat /tmp/tmp.sp7ciTlRyb ++ rm /tmp/tmp.4pbvFJztBO /tmp/tmp.sp7ciTlRyb ++ return 0 + client_pod=pxc-client-67fc4995bb-mb66m + wait_pod pxc-client-67fc4995bb-mb66m + local pod=pxc-client-67fc4995bb-mb66m + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-67fc4995bb-mb66m + local container= + set +o xtrace pod/pxc-client-67fc4995bb-mb66m condition met waiting for pod/pxc-client-67fc4995bb-mb66m to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.20mrecv5xV +++ mktemp ++ local LAST_ERR=/tmp/tmp.lgd03qt3vk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.20mrecv5xV ++ cat /tmp/tmp.lgd03qt3vk ++ rm /tmp/tmp.20mrecv5xV /tmp/tmp.lgd03qt3vk ++ return 0 + client_pod=pxc-client-67fc4995bb-mb66m + wait_pod pxc-client-67fc4995bb-mb66m + local pod=pxc-client-67fc4995bb-mb66m + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-mb66m ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-mb66m condition met waiting for pod/pxc-client-67fc4995bb-mb66m to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + grep -E 'server request payload|unary call' + grep -Eo '\{.*\}' + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.kubeVersion)' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-cr-76d4659869-vtjzk ++ mktemp + local LAST_OUT=/tmp/tmp.7p2oPS0kTY ++ mktemp + local LAST_ERR=/tmp/tmp.A8XiAK0MF9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-76d4659869-vtjzk + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7p2oPS0kTY + cat /tmp/tmp.A8XiAK0MF9 + rm /tmp/tmp.7p2oPS0kTY /tmp/tmp.A8XiAK0MF9 + return 0 + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.kubeVersion)' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + grep -Eo '\{.*\}' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.time_ms")' + grep -E 'server request payload|unary call' + kubectl_bin logs version-service-559c4df55c-pjvtg ++ mktemp + local LAST_OUT=/tmp/tmp.F7ZRNVp59U ++ mktemp + local LAST_ERR=/tmp/tmp.2OpCTJFXh1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-559c4df55c-pjvtg + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.F7ZRNVp59U + cat /tmp/tmp.2OpCTJFXh1 + rm /tmp/tmp.F7ZRNVp59U /tmp/tmp.2OpCTJFXh1 + return 0 + local telemetry_log_file=disabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' disabled == disabled -a disabled == enabled ']' + local image_prefix=disabled + image_prefix=disabled + local telemetry_cr_log_file=disabled_telemetry.version-service-cr-disabled-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + [[ disabled == \d\i\s\a\b\l\e\d\-\r\e\c\o\m\m\e\n\d\e\d ]] + [[ disabled == \d\i\s\a\b\l\e\d\-\l\a\t\e\s\t ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' disabled == disabled -a disabled == disabled ']' + desc 'CR VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- CR VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.eYNH1l0X9V/disabled_telemetry.version-service-cr.log.json ]] + desc 'operator VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.eYNH1l0X9V/disabled_telemetry.version-service.log.json ]] + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.b0hLhiUM3r ++ mktemp + local LAST_ERR=/tmp/tmp.sg3eDUHNWv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.b0hLhiUM3r perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.sg3eDUHNWv + rm /tmp/tmp.b0hLhiUM3r /tmp/tmp.sg3eDUHNWv + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.xgCBCOXxYZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.aCpXNCLmU9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xgCBCOXxYZ ++ cat /tmp/tmp.aCpXNCLmU9 ++ rm /tmp/tmp.xgCBCOXxYZ /tmp/tmp.aCpXNCLmU9 ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-698c684f5f-xn745 ++ mktemp + local LAST_OUT=/tmp/tmp.esi6TWmh2L ++ mktemp + local LAST_ERR=/tmp/tmp.cGOdWdmqpF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-698c684f5f-xn745 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.esi6TWmh2L pod "percona-xtradb-cluster-operator-698c684f5f-xn745" deleted from pxc-operator namespace + cat /tmp/tmp.cGOdWdmqpF + rm /tmp/tmp.esi6TWmh2L /tmp/tmp.cGOdWdmqpF + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.qOnBmKAQnD ++ mktemp + local LAST_ERR=/tmp/tmp.2CsVvN2LDb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qOnBmKAQnD perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-14904 namespace + cat /tmp/tmp.2CsVvN2LDb + rm /tmp/tmp.qOnBmKAQnD /tmp/tmp.2CsVvN2LDb + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.DpfzcHit1D ++ mktemp + local LAST_ERR=/tmp/tmp.PFOb1ZejnS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DpfzcHit1D deployment.apps "pxc-client" deleted from smart-update2-14904 namespace + cat /tmp/tmp.PFOb1ZejnS + rm /tmp/tmp.DpfzcHit1D /tmp/tmp.PFOb1ZejnS + return 0 + sleep 30 + kubectl_bin delete deployment version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.DBLedUYjO0 ++ mktemp + local LAST_ERR=/tmp/tmp.iQWMeCcJ83 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deployment version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DBLedUYjO0 deployment.apps "version-service-cr" deleted from smart-update2-14904 namespace + cat /tmp/tmp.iQWMeCcJ83 + rm /tmp/tmp.DBLedUYjO0 /tmp/tmp.iQWMeCcJ83 + return 0 + desc 'Telemetry testing finished' + set +o xtrace ----------------------------------------------------------------------------------- Telemetry testing finished ----------------------------------------------------------------------------------- + desc 'PXC cluster with version service offline' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster with version service offline ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/smart-update-version-service-unreachable.yml /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe"' /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + spinup_pxc smart-update /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.L4iioiQAWs ++ mktemp + local LAST_ERR=/tmp/tmp.O40JfXnxTP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.L4iioiQAWs secret/my-cluster-secrets created secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.O40JfXnxTP + rm /tmp/tmp.L4iioiQAWs /tmp/tmp.O40JfXnxTP + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + local LAST_OUT=/tmp/tmp.bV9qPhFouf + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-14904~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' ++ mktemp + local LAST_ERR=/tmp/tmp.eYuoiaVCng + local exit_status=0 + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bV9qPhFouf deployment.apps/pxc-client created + cat /tmp/tmp.eYuoiaVCng + rm /tmp/tmp.bV9qPhFouf /tmp/tmp.eYuoiaVCng + return 0 + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + apply_config /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local config_file=/tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local pvc_name= + '[' -z '' ']' + cat_config /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml '' + local input_file=/tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local pvc_name= + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + cat /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-14904~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#' + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + local LAST_OUT=/tmp/tmp.5n3ghw5ywO ++ mktemp + local LAST_ERR=/tmp/tmp.ajQSt4Mvgy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5n3ghw5ywO perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.ajQSt4Mvgy + rm /tmp/tmp.5n3ghw5ywO /tmp/tmp.ajQSt4Mvgy + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HxlJO2O5iq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4fstXOlGTu +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.HxlJO2O5iq +++ cat /tmp/tmp.4fstXOlGTu +++ rm /tmp/tmp.HxlJO2O5iq /tmp/tmp.4fstXOlGTu +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-14904 ++ mktemp + local LAST_OUT=/tmp/tmp.wdeK4IaA8b ++ mktemp + local LAST_ERR=/tmp/tmp.eiisE2lgSK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-14904 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-14904 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-14904 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.wdeK4IaA8b + cat /tmp/tmp.eiisE2lgSK error: no matching resources found + rm /tmp/tmp.wdeK4IaA8b /tmp/tmp.eiisE2lgSK + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace Error from server (NotFound): pods "smart-update-haproxy-0" not found waiting for pod/smart-update-haproxy-0 to become Ready......................................Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.nkaZ25XSL1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.v7CLkkBi30 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nkaZ25XSL1 ++ cat /tmp/tmp.v7CLkkBi30 ++ rm /tmp/tmp.nkaZ25XSL1 /tmp/tmp.v7CLkkBi30 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.u1lZVBdZpj +++ mktemp ++ local LAST_ERR=/tmp/tmp.QpA7cUTtyo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u1lZVBdZpj ++ cat /tmp/tmp.QpA7cUTtyo ++ rm /tmp/tmp.u1lZVBdZpj /tmp/tmp.QpA7cUTtyo ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2EQRZ6EoP7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.9S8KAk1S23 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2EQRZ6EoP7 ++ cat /tmp/tmp.9S8KAk1S23 ++ rm /tmp/tmp.2EQRZ6EoP7 /tmp/tmp.9S8KAk1S23 ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-t8f7z + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V2GxpQUr68 +++ mktemp ++ local LAST_ERR=/tmp/tmp.23qMGmQdzS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.V2GxpQUr68 ++ cat /tmp/tmp.23qMGmQdzS ++ rm /tmp/tmp.V2GxpQUr68 /tmp/tmp.23qMGmQdzS ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Jf0MZZcFh1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.2t4H6ZpkhJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Jf0MZZcFh1 ++ cat /tmp/tmp.2t4H6ZpkhJ ++ rm /tmp/tmp.Jf0MZZcFh1 /tmp/tmp.2t4H6ZpkhJ ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5Ca8ZkN7Ce +++ mktemp ++ local LAST_ERR=/tmp/tmp.nmofLczzoY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5Ca8ZkN7Ce ++ cat /tmp/tmp.nmofLczzoY ++ rm /tmp/tmp.5Ca8ZkN7Ce /tmp/tmp.nmofLczzoY ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + kubectl exec smart-update-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my + return 1 + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=300 + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BCuru7zfx7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.3KWjKh2ipp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BCuru7zfx7 ++ cat /tmp/tmp.3KWjKh2ipp ++ rm /tmp/tmp.BCuru7zfx7 /tmp/tmp.3KWjKh2ipp ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.u46IhF8tUg +++ mktemp ++ local LAST_ERR=/tmp/tmp.ss5r33Wk4T ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u46IhF8tUg ++ cat /tmp/tmp.ss5r33Wk4T ++ rm /tmp/tmp.u46IhF8tUg /tmp/tmp.ss5r33Wk4T ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.0KSJrtfgBg ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.02nRpmPb5F +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.0KSJrtfgBg +++++ cat /tmp/tmp.02nRpmPb5F +++++ rm /tmp/tmp.0KSJrtfgBg /tmp/tmp.02nRpmPb5F +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bMon7kW3w7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JJk3X9OmPS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bMon7kW3w7 ++ cat /tmp/tmp.JJk3X9OmPS ++ rm /tmp/tmp.bMon7kW3w7 /tmp/tmp.JJk3X9OmPS ++ return 0 + [[ 2 == \2 ]] + echo ++ kubectl_bin get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ur3k9fZNE8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.BnUDd7Xt9l ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ur3k9fZNE8 ++ cat /tmp/tmp.BnUDd7Xt9l ++ rm /tmp/tmp.Ur3k9fZNE8 /tmp/tmp.BnUDd7Xt9l ++ return 0 + [[ percona/percona-xtradb-cluster:8.4.5-5.1 != \p\e\r\c\o\n\a\/\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\:\8\.\4\.\5\-\5\.\1 ]] + desc 'PXC cluster update with latest image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with latest image by version service ----------------------------------------------------------------------------------- + vs_image=latest ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jONx3hvu93 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.s8JzZ22457 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.jONx3hvu93 +++ cat /tmp/tmp.s8JzZ22457 +++ rm /tmp/tmp.jONx3hvu93 /tmp/tmp.s8JzZ22457 +++ return 0 ++ client_pod=pxc-client-58956d9c99-t8f7z ++ wait_pod pxc-client-58956d9c99-t8f7z ++ local pod=pxc-client-58956d9c99-t8f7z ++ local max_retry=480 ++ local ns= +++ echo pxc-client-58956d9c99-t8f7z +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ grep -E '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.dZKZRL8yoE ++ mktemp + local LAST_ERR=/tmp/tmp.drQvAAScYa + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dZKZRL8yoE perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.drQvAAScYa + rm /tmp/tmp.dZKZRL8yoE /tmp/tmp.drQvAAScYa + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=300 + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tLLXXC1UVR +++ mktemp ++ local LAST_ERR=/tmp/tmp.hgzYFFIJJy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tLLXXC1UVR ++ cat /tmp/tmp.hgzYFFIJJy ++ rm /tmp/tmp.tLLXXC1UVR /tmp/tmp.hgzYFFIJJy ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.N5w709RFk1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.9ces9h5JpG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.N5w709RFk1 ++ cat /tmp/tmp.9ces9h5JpG ++ rm /tmp/tmp.N5w709RFk1 /tmp/tmp.9ces9h5JpG ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Ygtx6XlXQZ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.HsKyNwQE84 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Ygtx6XlXQZ +++++ cat /tmp/tmp.HsKyNwQE84 +++++ rm /tmp/tmp.Ygtx6XlXQZ /tmp/tmp.HsKyNwQE84 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qwdtTkrLI4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.bNGWNSBkcI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qwdtTkrLI4 ++ cat /tmp/tmp.bNGWNSBkcI ++ rm /tmp/tmp.qwdtTkrLI4 /tmp/tmp.bNGWNSBkcI ++ return 0 + [[ 2 == \2 ]] + echo ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Nhk0ee5RMb +++ mktemp ++ local LAST_ERR=/tmp/tmp.cGWasW5SaE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Nhk0ee5RMb ++ cat /tmp/tmp.cGWasW5SaE ++ rm /tmp/tmp.Nhk0ee5RMb /tmp/tmp.cGWasW5SaE ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tb8czwbCD2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JB2MwBenhi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tb8czwbCD2 ++ cat /tmp/tmp.JB2MwBenhi ++ rm /tmp/tmp.tb8czwbCD2 /tmp/tmp.JB2MwBenhi ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-t8f7z + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QDWC8Sv0VW +++ mktemp ++ local LAST_ERR=/tmp/tmp.F8MNyXkbD8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QDWC8Sv0VW ++ cat /tmp/tmp.F8MNyXkbD8 ++ rm /tmp/tmp.QDWC8Sv0VW /tmp/tmp.F8MNyXkbD8 ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-58956d9c99-t8f7z ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + kubectl_bin delete -f /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.o4XiH9kN7G ++ mktemp + local LAST_ERR=/tmp/tmp.CoKodzV77I + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.o4XiH9kN7G perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-14904 namespace + cat /tmp/tmp.CoKodzV77I + rm /tmp/tmp.o4XiH9kN7G /tmp/tmp.CoKodzV77I + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.t7l3Xfh2mp ++ mktemp + local LAST_ERR=/tmp/tmp.8rBxi1GdmF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.t7l3Xfh2mp persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-14904 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-14904 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-14904 namespace + cat /tmp/tmp.8rBxi1GdmF + rm /tmp/tmp.t7l3Xfh2mp /tmp/tmp.8rBxi1GdmF + return 0 + desc 'PXC cluster update with the latest image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with the latest image by version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.oDDJbkaKSM ++ mktemp + local LAST_ERR=/tmp/tmp.DTj47pvTXg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oDDJbkaKSM secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.DTj47pvTXg + rm /tmp/tmp.oDDJbkaKSM /tmp/tmp.DTj47pvTXg + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-14904~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.lyLOx7b91a ++ mktemp + local LAST_ERR=/tmp/tmp.mZdSH6a7gF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lyLOx7b91a deployment.apps/pxc-client unchanged + cat /tmp/tmp.mZdSH6a7gF + rm /tmp/tmp.lyLOx7b91a /tmp/tmp.mZdSH6a7gF + return 0 + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + apply_config /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local config_file=/tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local pvc_name= + '[' -z '' ']' + cat_config /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml '' + local input_file=/tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local pvc_name= + kubectl_bin apply -f - + cat /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#' + local LAST_OUT=/tmp/tmp.IiUKgkqd7a ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_ERR=/tmp/tmp.zRZ4oMZLDc + local exit_status=0 + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-14904~ + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ seq 0 2 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IiUKgkqd7a perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.zRZ4oMZLDc + rm /tmp/tmp.IiUKgkqd7a /tmp/tmp.zRZ4oMZLDc + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.q5qn2GI1Pr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fjxX9r33hu +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.q5qn2GI1Pr +++ cat /tmp/tmp.fjxX9r33hu +++ rm /tmp/tmp.q5qn2GI1Pr /tmp/tmp.fjxX9r33hu +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-14904 ++ mktemp + local LAST_OUT=/tmp/tmp.mR9ogyMUnA ++ mktemp + local LAST_ERR=/tmp/tmp.uOrKeQkLsj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-14904 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-14904 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-14904 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.mR9ogyMUnA + cat /tmp/tmp.uOrKeQkLsj error: no matching resources found + rm /tmp/tmp.mR9ogyMUnA /tmp/tmp.uOrKeQkLsj + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met waiting for pod/smart-update-haproxy-0 to become Ready.Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.A5vmNkUMOv +++ mktemp ++ local LAST_ERR=/tmp/tmp.TmaAjk4WkT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A5vmNkUMOv ++ cat /tmp/tmp.TmaAjk4WkT ++ rm /tmp/tmp.A5vmNkUMOv /tmp/tmp.TmaAjk4WkT ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gOxE1qMkcb +++ mktemp ++ local LAST_ERR=/tmp/tmp.mC6GTgHGwj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gOxE1qMkcb ++ cat /tmp/tmp.mC6GTgHGwj ++ rm /tmp/tmp.gOxE1qMkcb /tmp/tmp.mC6GTgHGwj ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UnIJENAxEB +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z7x40a0NJG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UnIJENAxEB ++ cat /tmp/tmp.Z7x40a0NJG ++ rm /tmp/tmp.UnIJENAxEB /tmp/tmp.Z7x40a0NJG ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DTgk1L1vc7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.tEGn1lSz8f ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DTgk1L1vc7 ++ cat /tmp/tmp.tEGn1lSz8f ++ rm /tmp/tmp.DTgk1L1vc7 /tmp/tmp.tEGn1lSz8f ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ReQ2vcfTLx +++ mktemp ++ local LAST_ERR=/tmp/tmp.T4UGwkJ7SH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ReQ2vcfTLx ++ cat /tmp/tmp.T4UGwkJ7SH ++ rm /tmp/tmp.ReQ2vcfTLx /tmp/tmp.T4UGwkJ7SH ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cGjvU6jwXU +++ mktemp ++ local LAST_ERR=/tmp/tmp.EZtliCbHYG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cGjvU6jwXU ++ cat /tmp/tmp.EZtliCbHYG ++ rm /tmp/tmp.cGjvU6jwXU /tmp/tmp.EZtliCbHYG ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + kubectl exec smart-update-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my + return 1 + vs_image=latest ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.n8sjEcKawJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.88ZSZt8bBI +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.n8sjEcKawJ +++ cat /tmp/tmp.88ZSZt8bBI +++ rm /tmp/tmp.n8sjEcKawJ /tmp/tmp.88ZSZt8bBI +++ return 0 ++ client_pod=pxc-client-58956d9c99-t8f7z ++ wait_pod pxc-client-58956d9c99-t8f7z ++ local pod=pxc-client-58956d9c99-t8f7z ++ local max_retry=480 ++ local ns= +++ echo pxc-client-58956d9c99-t8f7z +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ grep -E '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.jYtmKAMXWT ++ mktemp + local LAST_ERR=/tmp/tmp.iyotOsIt5B + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jYtmKAMXWT perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.iyotOsIt5B + rm /tmp/tmp.jYtmKAMXWT /tmp/tmp.iyotOsIt5B + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.4 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=300 + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HKRUE7Wum8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.D7jyBvRfad ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HKRUE7Wum8 ++ cat /tmp/tmp.D7jyBvRfad ++ rm /tmp/tmp.HKRUE7Wum8 /tmp/tmp.D7jyBvRfad ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oLBRXYZAZZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.9LOnfiPjjC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oLBRXYZAZZ ++ cat /tmp/tmp.9LOnfiPjjC ++ rm /tmp/tmp.oLBRXYZAZZ /tmp/tmp.9LOnfiPjjC ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.UYp8g1mVlS ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.AJautCm0kd +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.UYp8g1mVlS +++++ cat /tmp/tmp.AJautCm0kd +++++ rm /tmp/tmp.UYp8g1mVlS /tmp/tmp.AJautCm0kd +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.puXAlbTBdn +++ mktemp ++ local LAST_ERR=/tmp/tmp.nksNn2UNGQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.puXAlbTBdn ++ cat /tmp/tmp.nksNn2UNGQ ++ rm /tmp/tmp.puXAlbTBdn /tmp/tmp.nksNn2UNGQ ++ return 0 + [[ 2 == \2 ]] + echo ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Oe37ZTGGHC +++ mktemp ++ local LAST_ERR=/tmp/tmp.zTH3VbzvRA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Oe37ZTGGHC ++ cat /tmp/tmp.zTH3VbzvRA ++ rm /tmp/tmp.Oe37ZTGGHC /tmp/tmp.zTH3VbzvRA ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-58956d9c99-t8f7z + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.p0VtgiDjq2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.B7vG5SnZ1r ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.p0VtgiDjq2 ++ cat /tmp/tmp.B7vG5SnZ1r ++ rm /tmp/tmp.p0VtgiDjq2 /tmp/tmp.B7vG5SnZ1r ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yUY8JmZKgi +++ mktemp ++ local LAST_ERR=/tmp/tmp.B3MT64K09h ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yUY8JmZKgi ++ cat /tmp/tmp.B3MT64K09h ++ rm /tmp/tmp.yUY8JmZKgi /tmp/tmp.B3MT64K09h ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + kubectl_bin delete -f /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.stDZpUYGg2 ++ mktemp + local LAST_ERR=/tmp/tmp.ag2hqOnKSM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.stDZpUYGg2 perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-14904 namespace + cat /tmp/tmp.ag2hqOnKSM + rm /tmp/tmp.stDZpUYGg2 /tmp/tmp.ag2hqOnKSM + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.uE2R3ITW4b ++ mktemp + local LAST_ERR=/tmp/tmp.jjDh9VFOZy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uE2R3ITW4b persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-14904 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-14904 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-14904 namespace + cat /tmp/tmp.jjDh9VFOZy + rm /tmp/tmp.uE2R3ITW4b /tmp/tmp.jjDh9VFOZy + return 0 + desc 'PXC cluster update with explicitly specified image inside version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with explicitly specified image inside version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.XR2Nw2Ak1E ++ mktemp + local LAST_ERR=/tmp/tmp.oSPpzVqBYw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XR2Nw2Ak1E secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.oSPpzVqBYw + rm /tmp/tmp.XR2Nw2Ak1E /tmp/tmp.oSPpzVqBYw + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + local LAST_OUT=/tmp/tmp.5j4TZZ83ic + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-14904~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_ERR=/tmp/tmp.8JP1pC8XAB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5j4TZZ83ic deployment.apps/pxc-client unchanged + cat /tmp/tmp.8JP1pC8XAB + rm /tmp/tmp.5j4TZZ83ic /tmp/tmp.8JP1pC8XAB + return 0 + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + apply_config /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local config_file=/tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml '' + local input_file=/tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + local pvc_name= + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-14904~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + cat /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#' + local LAST_OUT=/tmp/tmp.EG833XAiLH ++ mktemp + local LAST_ERR=/tmp/tmp.2cvchve8La + local exit_status=0 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EG833XAiLH perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.2cvchve8La + rm /tmp/tmp.EG833XAiLH /tmp/tmp.2cvchve8La + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.F1XoIt4AZY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kkL4eRNDGK +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.F1XoIt4AZY +++ cat /tmp/tmp.kkL4eRNDGK +++ rm /tmp/tmp.F1XoIt4AZY /tmp/tmp.kkL4eRNDGK +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-14904 ++ mktemp + local LAST_OUT=/tmp/tmp.euyOaMkClm ++ mktemp + local LAST_ERR=/tmp/tmp.KpzbrCA9gj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-14904 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-14904 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-14904 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.euyOaMkClm + cat /tmp/tmp.KpzbrCA9gj error: no matching resources found + rm /tmp/tmp.euyOaMkClm /tmp/tmp.KpzbrCA9gj + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met waiting for pod/smart-update-haproxy-0 to become Ready.Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.7kfnuPOwif +++ mktemp ++ local LAST_ERR=/tmp/tmp.28K7BZXAPH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7kfnuPOwif ++ cat /tmp/tmp.28K7BZXAPH ++ rm /tmp/tmp.7kfnuPOwif /tmp/tmp.28K7BZXAPH ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TyummzhmqR +++ mktemp ++ local LAST_ERR=/tmp/tmp.kNiUREYYU2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TyummzhmqR ++ cat /tmp/tmp.kNiUREYYU2 ++ rm /tmp/tmp.TyummzhmqR /tmp/tmp.kNiUREYYU2 ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f7EtC9voKP +++ mktemp ++ local LAST_ERR=/tmp/tmp.5OvJGGAnjb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.f7EtC9voKP ++ cat /tmp/tmp.5OvJGGAnjb ++ rm /tmp/tmp.f7EtC9voKP /tmp/tmp.5OvJGGAnjb ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z9ofnRrXbH +++ mktemp ++ local LAST_ERR=/tmp/tmp.jmnf2cofIb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Z9ofnRrXbH ++ cat /tmp/tmp.jmnf2cofIb ++ rm /tmp/tmp.Z9ofnRrXbH /tmp/tmp.jmnf2cofIb ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z6nFa2nSBn +++ mktemp ++ local LAST_ERR=/tmp/tmp.5oQDd5GOfC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Z6nFa2nSBn ++ cat /tmp/tmp.5oQDd5GOfC ++ rm /tmp/tmp.Z6nFa2nSBn /tmp/tmp.5oQDd5GOfC ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.H8itRAulQe +++ mktemp ++ local LAST_ERR=/tmp/tmp.jYj4R4CFz8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.H8itRAulQe ++ cat /tmp/tmp.jYj4R4CFz8 ++ rm /tmp/tmp.H8itRAulQe /tmp/tmp.jYj4R4CFz8 ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + is_keyring_plugin_in_use smart-update + local cluster=smart-update + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + kubectl exec smart-update-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my + return 1 ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ grep :8.4 +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++ sort -V ++ head -n1 ++ tail -n2 ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++++ grep -c percona-xtradb-cluster-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ head -1 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zbC0QTy5Jo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xSgbIJ7egm +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.zbC0QTy5Jo +++ cat /tmp/tmp.xSgbIJ7egm +++ rm /tmp/tmp.zbC0QTy5Jo /tmp/tmp.xSgbIJ7egm +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-698c684f5f-f4589 -n pxc-operator -- curl -s http://version-service.smart-update2-14904.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.6gCtxRvnks +++ mktemp ++ local LAST_ERR=/tmp/tmp.22D6l5K8sa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-698c684f5f-f4589 -n pxc-operator -- curl -s http://version-service.smart-update2-14904.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6gCtxRvnks ++ cat /tmp/tmp.22D6l5K8sa Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.6gCtxRvnks /tmp/tmp.22D6l5K8sa ++ return 0 + vs_image=percona/percona-xtradb-cluster:8.4.5-5.1 ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8BMuEwnVNw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.pqv5z0MiQK +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.8BMuEwnVNw +++ cat /tmp/tmp.pqv5z0MiQK +++ rm /tmp/tmp.8BMuEwnVNw /tmp/tmp.pqv5z0MiQK +++ return 0 ++ client_pod=pxc-client-58956d9c99-t8f7z ++ wait_pod pxc-client-58956d9c99-t8f7z ++ local pod=pxc-client-58956d9c99-t8f7z ++ local max_retry=480 ++ local ns= +++ echo pxc-client-58956d9c99-t8f7z +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ grep -E '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.4.5-5.1","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.aamuB56CTe ++ mktemp + local LAST_ERR=/tmp/tmp.FoE1aq1U0s + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.4.5-5.1","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.aamuB56CTe perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.FoE1aq1U0s + rm /tmp/tmp.aamuB56CTe /tmp/tmp.FoE1aq1U0s + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.4.5-5.1 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.4.5-5.1 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=300 + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bxww9Bewpi +++ mktemp ++ local LAST_ERR=/tmp/tmp.UGf5DFmb7O ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bxww9Bewpi ++ cat /tmp/tmp.UGf5DFmb7O ++ rm /tmp/tmp.bxww9Bewpi /tmp/tmp.UGf5DFmb7O ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ej4Xszs3Pb +++ mktemp ++ local LAST_ERR=/tmp/tmp.slzLQszNSY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ej4Xszs3Pb ++ cat /tmp/tmp.slzLQszNSY ++ rm /tmp/tmp.Ej4Xszs3Pb /tmp/tmp.slzLQszNSY ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.BHyDSZM2QO ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.eQBMHY3uUW +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.BHyDSZM2QO +++++ cat /tmp/tmp.eQBMHY3uUW +++++ rm /tmp/tmp.BHyDSZM2QO /tmp/tmp.eQBMHY3uUW +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lQo817AFS5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.1yvFJB25HP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lQo817AFS5 ++ cat /tmp/tmp.1yvFJB25HP ++ rm /tmp/tmp.lQo817AFS5 /tmp/tmp.1yvFJB25HP ++ return 0 + [[ 2 == \2 ]] + echo ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.l7NfC2uNHL +++ mktemp ++ local LAST_ERR=/tmp/tmp.C1IhPrLuOH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.l7NfC2uNHL ++ cat /tmp/tmp.C1IhPrLuOH ++ rm /tmp/tmp.l7NfC2uNHL /tmp/tmp.C1IhPrLuOH ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vZ55Wgm5va +++ mktemp ++ local LAST_ERR=/tmp/tmp.dDd0BZ1zv8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vZ55Wgm5va ++ cat /tmp/tmp.dDd0BZ1zv8 ++ rm /tmp/tmp.vZ55Wgm5va /tmp/tmp.dDd0BZ1zv8 ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-84.sql ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]] + [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fxOv32A6FO +++ mktemp ++ local LAST_ERR=/tmp/tmp.eqLEAyVNNF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fxOv32A6FO ++ cat /tmp/tmp.eqLEAyVNNF ++ rm /tmp/tmp.fxOv32A6FO /tmp/tmp.eqLEAyVNNF ++ return 0 + client_pod=pxc-client-58956d9c99-t8f7z + wait_pod pxc-client-58956d9c99-t8f7z + local pod=pxc-client-58956d9c99-t8f7z + local max_retry=480 + local ns= ++ echo pxc-client-58956d9c99-t8f7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-58956d9c99-t8f7z condition met waiting for pod/pxc-client-58956d9c99-t8f7z to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.eYNH1l0X9V/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.eYNH1l0X9V/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.eYNH1l0X9V/select-1.sql + kubectl_bin delete -f /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.LwkCZJQk55 ++ mktemp + local LAST_ERR=/tmp/tmp.t88ZcHTxeo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.eYNH1l0X9V/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LwkCZJQk55 perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-14904 namespace + cat /tmp/tmp.t88ZcHTxeo + rm /tmp/tmp.LwkCZJQk55 /tmp/tmp.t88ZcHTxeo + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.eyAvh1eCyv ++ mktemp + local LAST_ERR=/tmp/tmp.LmW2HrC7Rs + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eyAvh1eCyv persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-14904 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-14904 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-14904 namespace + cat /tmp/tmp.LmW2HrC7Rs + rm /tmp/tmp.eyAvh1eCyv /tmp/tmp.LmW2HrC7Rs + return 0 + desc cleanup + set +o xtrace ----------------------------------------------------------------------------------- cleanup ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.kn99AEIW63 ++ mktemp + local LAST_ERR=/tmp/tmp.Lc45snBYq8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kn99AEIW63 deployment.apps "version-service" deleted from smart-update2-14904 namespace service "version-service" deleted from smart-update2-14904 namespace + cat /tmp/tmp.Lc45snBYq8 + rm /tmp/tmp.kn99AEIW63 /tmp/tmp.Lc45snBYq8 + return 0 + destroy smart-update2-14904 + local namespace=smart-update2-14904 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'get backup status: Job.batch' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ + grep -v 'the object has been modified' + sort -u +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator + tee /tmp/tmp.eYNH1l0X9V/operator.log + grep -v level=info ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TmdYUDP2LL +++ mktemp ++ local LAST_ERR=/tmp/tmp.wNZVpUUR2S ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TmdYUDP2LL ++ cat /tmp/tmp.wNZVpUUR2S ++ rm /tmp/tmp.TmdYUDP2LL /tmp/tmp.wNZVpUUR2S ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-698c684f5f-f4589 ++ mktemp + local LAST_OUT=/tmp/tmp.sQiKFI3JOf ++ mktemp + local LAST_ERR=/tmp/tmp.fJu3Lxb2zx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-698c684f5f-f4589 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sQiKFI3JOf + cat /tmp/tmp.fJu3Lxb2zx + rm /tmp/tmp.sQiKFI3JOf /tmp/tmp.fJu3Lxb2zx + return 0 2026/04/09 04:29:16 http: TLS handshake error from 10.122.170.3:58412: EOF 2026-04-09T04:28:55.083Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.32.13-gke.1205000"} 2026-04-09T04:28:55.084Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-04-09T04:28:55.084Z INFO setup Manager starting up {"gitCommit": "3b65d7fe6a0a5db845238d34252d90db1f886e1d", "gitBranch": "PR-2434-3b65d7fe", "buildTime": "2026-04-09T01:11:10Z", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"} 2026-04-09T04:28:55.087Z INFO setup Registering Components. 2026-04-09T04:28:55.763Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-04-09T04:28:55.763Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-04-09T04:28:55.763Z INFO controller-runtime.metrics Starting metrics server 2026-04-09T04:28:55.763Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-04-09T04:28:55.763Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-04-09T04:28:55.763Z INFO controller-runtime.webhook Starting webhook server 2026-04-09T04:28:55.763Z INFO setup Starting the Cmd. 2026-04-09T04:28:55.763Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-04-09T04:28:55.764Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-04-09T04:28:55.864Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-04-09T04:29:15.591Z DEBUG events percona-xtradb-cluster-operator-698c684f5f-f4589_4bdb13a4-2cdf-48ee-befc-b98e0bd218a9 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"5b1a6e27-6fdc-4cac-86d2-1ad4d255c56a","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1775708955583663009"}, "reason": "LeaderElection"} 2026-04-09T04:29:15.591Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-04-09T04:29:15.591Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-04-09T04:29:15.591Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-04-09T04:29:15.591Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-04-09T04:29:15.591Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-04-09T04:29:15.692Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-04-09T04:29:15.692Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-04-09T04:29:15.692Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-04-09T04:29:15.692Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-04-09T04:29:15.792Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-04-09T04:29:15.792Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-04-09T04:29:16.349Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"minimal-cluster","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "minimal-cluster", "reconcileID": "6ba3cd0c-dfb9-4923-a849-6659ba53c636", "error": "failed to update cr finalizers: Internal error occurred: failed calling webhook \"validationwebhook.pxc.percona.com\": failed to call webhook: Post \"https://percona-xtradb-cluster-operator.pxc-operator.svc:443/validate-percona-xtradbcluster?timeout=10s\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \"crypto/rsa: verification error\" while trying to verify candidate authority certificate \"Root CA\")", "errorVerbose": "Internal error occurred: failed calling webhook \"validationwebhook.pxc.percona.com\": failed to call webhook: Post \"https://percona-xtradb-cluster-operator.pxc-operator.svc:443/validate-percona-xtradbcluster?timeout=10s\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \"crypto/rsa: verification error\" while trying to verify candidate authority certificate \"Root CA\")\nfailed to update cr finalizers\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} 2026-04-09T04:29:20.961Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"minimal-cluster","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "minimal-cluster", "reconcileID": "027f3440-edcc-4ccb-bc24-8f4b2b83501a", "error": "PerconaXtraDBCluster.pxc.percona.com \"minimal-cluster\" not found"} 2026-04-09T04:29:55.888Z INFO spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`. {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "2070fceb-0022-47fe-a58a-55619c057652"} 2026-04-09T04:29:58.946Z INFO spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`. {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "2070fceb-0022-47fe-a58a-55619c057652"} 2026-04-09T04:29:58.975Z INFO spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`. {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "2070fceb-0022-47fe-a58a-55619c057652"} 2026-04-09T04:30:02.077Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "2070fceb-0022-47fe-a58a-55619c057652", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-04-09T04:30:02.095Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "2070fceb-0022-47fe-a58a-55619c057652", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-04-09T04:30:02.663Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "2070fceb-0022-47fe-a58a-55619c057652", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:92\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:25\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:27\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:50\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:543\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:556\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} 2026-04-09T04:30:02.770Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "c8450768-c3e1-4ab0-8ee2-efca83b2db22", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-09T04:30:02.819Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "c8450768-c3e1-4ab0-8ee2-efca83b2db22", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-09T04:30:02.978Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "c8450768-c3e1-4ab0-8ee2-efca83b2db22", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:30:03.015Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "c8450768-c3e1-4ab0-8ee2-efca83b2db22", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:30:03.076Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "c8450768-c3e1-4ab0-8ee2-efca83b2db22", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:30:03.189Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "c8450768-c3e1-4ab0-8ee2-efca83b2db22", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:30:03.649Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "9ffb25d7-f5c9-4bf0-83d3-0e487e1cba94", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-09T04:30:03.674Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "9ffb25d7-f5c9-4bf0-83d3-0e487e1cba94", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-09T04:31:25.896Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "8beb73f8-7f95-4bd6-af35-8b896be14f40", "user": "operator"} 2026-04-09T04:31:25.931Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "8beb73f8-7f95-4bd6-af35-8b896be14f40", "user": "monitor"} 2026-04-09T04:31:25.976Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "8beb73f8-7f95-4bd6-af35-8b896be14f40"} 2026-04-09T04:31:26.005Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "8beb73f8-7f95-4bd6-af35-8b896be14f40"} 2026-04-09T04:31:26.033Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "8beb73f8-7f95-4bd6-af35-8b896be14f40", "user": "xtrabackup"} 2026-04-09T04:31:26.072Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "8beb73f8-7f95-4bd6-af35-8b896be14f40"} 2026-04-09T04:31:26.103Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "8beb73f8-7f95-4bd6-af35-8b896be14f40", "user": "replication"} 2026-04-09T04:34:13.578Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "4c6c97f1-75d8-477e-bd73-50857a15e2af", "user": "root"} 2026-04-09T04:34:14.652Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "4c6c97f1-75d8-477e-bd73-50857a15e2af", "new version": "8.4.5-5.1"} 2026-04-09T04:35:49.178Z INFO add new job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "2a72de70-6c4c-41d3-af33-30a6c27e9c20", "name": "ensure-version/smart-update2-14904/smart-update", "schedule": "* * * * *"} 2026-04-09T04:36:49.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "2a72de70-6c4c-41d3-af33-30a6c27e9c20", "endpoint": "http://version-service.smart-update2-14904.svc.cluster.local:11000"} 2026-04-09T04:36:49.044Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "2a72de70-6c4c-41d3-af33-30a6c27e9c20", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=122458d5-0170-47fc-a78d-f339ae0f8a78&databaseVersion=8.4.5-5.1&kubeVersion=v1.32.13-gke.1205000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=122458d5-0170-47fc-a78d-f339ae0f8a78&databaseVersion=8.4.5-5.1&kubeVersion=v1.32.13-gke.1205000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:300\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:302\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} 2026-04-09T04:37:55.323Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "442f1d56-5e7a-4242-8fa5-a63c497d4608", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2026-04-09T04:37:55.323Z INFO cluster is not found, deleting the job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "2a72de70-6c4c-41d3-af33-30a6c27e9c20", "name": "ensure-version/smart-update2-14904/smart-update", "cluster": "smart-update", "namespace": "smart-update2-14904"} 2026-04-09T04:38:12.768Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "88df6a81-0372-404b-a9c0-56eb30ccfb32", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-04-09T04:38:12.785Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "88df6a81-0372-404b-a9c0-56eb30ccfb32", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-04-09T04:38:13.357Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "88df6a81-0372-404b-a9c0-56eb30ccfb32", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:92\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:25\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:27\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:50\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:543\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:556\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} 2026-04-09T04:38:13.470Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "b60f66e7-6221-440f-b733-bb84f8ff56c1", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-09T04:38:13.509Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "b60f66e7-6221-440f-b733-bb84f8ff56c1", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-09T04:38:13.558Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "b60f66e7-6221-440f-b733-bb84f8ff56c1", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:38:13.624Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "b60f66e7-6221-440f-b733-bb84f8ff56c1", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:38:13.688Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "b60f66e7-6221-440f-b733-bb84f8ff56c1", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:38:13.781Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "b60f66e7-6221-440f-b733-bb84f8ff56c1", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:38:14.580Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "56b20230-af7f-4598-9d99-380c3d2563a9", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-09T04:38:14.602Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "56b20230-af7f-4598-9d99-380c3d2563a9", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-09T04:42:12.114Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "915e9255-fc29-42fb-997e-134ac52adbf8", "new version": "8.4.5-5.1"} 2026-04-09T04:43:41.811Z INFO add new job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "7d6cb54d-8e44-4fa4-82d5-7b01668d7376", "name": "ensure-version/smart-update2-14904/smart-update", "schedule": "* * * * *"} 2026-04-09T04:44:41.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "7d6cb54d-8e44-4fa4-82d5-7b01668d7376", "endpoint": "http://version-service.smart-update2-14904.svc.cluster.local:11000"} 2026-04-09T04:44:41.052Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "7d6cb54d-8e44-4fa4-82d5-7b01668d7376", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=586d55b7-5f8b-49ad-a81c-47f463f30d3c&databaseVersion=8.4.5-5.1&kubeVersion=v1.32.13-gke.1205000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=586d55b7-5f8b-49ad-a81c-47f463f30d3c&databaseVersion=8.4.5-5.1&kubeVersion=v1.32.13-gke.1205000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:300\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:302\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} 2026-04-09T04:45:41.001Z INFO cluster is not ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "7d6cb54d-8e44-4fa4-82d5-7b01668d7376"} 2026-04-09T04:45:53.861Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "e685806b-3d62-4ee0-86af-ce194c8d8c09", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2026-04-09T04:46:09.007Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "95529164-2e4a-49ec-abd6-753f2c5e26ab", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-04-09T04:46:09.022Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "95529164-2e4a-49ec-abd6-753f2c5e26ab", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-04-09T04:46:09.579Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "95529164-2e4a-49ec-abd6-753f2c5e26ab", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:92\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:25\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:27\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:50\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:543\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:556\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} 2026-04-09T04:46:09.721Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "98340ecf-48f0-409f-afb2-391bde6bdb07", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-09T04:46:09.762Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "98340ecf-48f0-409f-afb2-391bde6bdb07", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-09T04:46:09.847Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "98340ecf-48f0-409f-afb2-391bde6bdb07", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:46:09.936Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "98340ecf-48f0-409f-afb2-391bde6bdb07", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:46:10.014Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "98340ecf-48f0-409f-afb2-391bde6bdb07", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:46:10.170Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "98340ecf-48f0-409f-afb2-391bde6bdb07", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-09T04:46:11.060Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "d2558bd7-b037-4217-9df9-4d6961fff5b0", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-09T04:46:11.083Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "d2558bd7-b037-4217-9df9-4d6961fff5b0", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-09T04:47:29.868Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "bc99bab1-af81-4030-b322-4e1dbed54210", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.235.90:3306: connect: connection refused"} 2026-04-09T04:50:10.913Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "748a39f3-1506-4b97-a465-010e099db455", "new version": "8.4.5-5.1"} 2026-04-09T04:51:38.994Z INFO add new job {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "60f8da70-a509-48c5-a27b-102b4f29ed71", "name": "ensure-version/smart-update2-14904/smart-update", "schedule": "* * * * *"} 2026-04-09T04:52:38.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "60f8da70-a509-48c5-a27b-102b4f29ed71", "endpoint": "http://version-service.smart-update2-14904.svc.cluster.local:11000"} 2026-04-09T04:52:38.043Z ERROR failed to ensure version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "60f8da70-a509-48c5-a27b-102b4f29ed71", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.4.5-5.1?clusterWideEnabled=true&customResourceUid=4e9b9b4e-7146-456e-a9e7-659172688aa9&databaseVersion=8.4.5-5.1&kubeVersion=v1.32.13-gke.1205000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.4.5-5.1?clusterWideEnabled=true&customResourceUid=4e9b9b4e-7146-456e-a9e7-659172688aa9&databaseVersion=8.4.5-5.1&kubeVersion=v1.32.13-gke.1205000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:300\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:302\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} 2026-04-09T04:53:38.001Z INFO cluster is not ready {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "60f8da70-a509-48c5-a27b-102b4f29ed71"} 2026-04-09T04:53:46.982Z ERROR Update status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-14904"}, "namespace": "smart-update2-14904", "name": "smart-update", "reconcileID": "8af529f9-894e-449c-bb80-5c45391d6c80", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1 github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1 github.com/robfig/cron/v3.(*Cron).startJob.func1 github.com/robfig/cron/v3.FuncJob.Run /go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136 /go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:495 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:259 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:484 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:214 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.7wW9zLi5H0 ++ mktemp + local LAST_ERR=/tmp/tmp.kBluX17Z6S + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7wW9zLi5H0 No resources found + cat /tmp/tmp.kBluX17Z6S + rm /tmp/tmp.7wW9zLi5H0 /tmp/tmp.kBluX17Z6S + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.4V7kkLBB98 ++ mktemp + local LAST_ERR=/tmp/tmp.QM6s7aUlBB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4V7kkLBB98 No resources found + cat /tmp/tmp.QM6s7aUlBB + rm /tmp/tmp.4V7kkLBB98 /tmp/tmp.QM6s7aUlBB + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.sNpNyJASob ++ mktemp + local LAST_ERR=/tmp/tmp.T302EPHemI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sNpNyJASob No resources found + cat /tmp/tmp.T302EPHemI + rm /tmp/tmp.sNpNyJASob /tmp/tmp.T302EPHemI + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.7NElBkRqT2 ++ mktemp + local LAST_ERR=/tmp/tmp.l6I0tWfvfY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7NElBkRqT2 validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.l6I0tWfvfY + rm /tmp/tmp.7NElBkRqT2 /tmp/tmp.l6I0tWfvfY + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace smart-update2-14904 + rm -rf /tmp/tmp.eYNH1l0X9V ++ mktemp + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ihVIlByG45 + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- ++ mktemp + local LAST_OUT=/tmp/tmp.xFrkH8OV6Z + local LAST_ERR=/tmp/tmp.xbM8l2IvQo + local exit_status=0 ++ mktemp ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace smart-update2-14904 + local LAST_ERR=/tmp/tmp.yAVlGJHrFy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator