Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/logs/smart-update2-8-0.log Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 + API=pxc.percona.com/v9-9-9 + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + CLUSTER=smart-update + CLUSTER_SIZE=3 + PROXY_SIZE=2 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + PXC_VER=8.0 + TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + VS_URL=http://version-service + VS_PORT=11000 + VS_ENDPOINT=http://version-service:11000 + main + create_infra smart-update2-10649 + local ns=smart-update2-10649 + '[' -n pxc-operator ']' + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.4RYK14Mrjv ++ mktemp + local LAST_ERR=/tmp/tmp.wkoCiCHc4F + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4RYK14Mrjv No resources found + cat /tmp/tmp.wkoCiCHc4F + rm /tmp/tmp.4RYK14Mrjv /tmp/tmp.wkoCiCHc4F + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.nfrHitCIzg ++ mktemp + local LAST_ERR=/tmp/tmp.ufeGq1qf5v + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nfrHitCIzg No resources found + cat /tmp/tmp.ufeGq1qf5v + rm /tmp/tmp.nfrHitCIzg /tmp/tmp.ufeGq1qf5v + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.r0aksH8xSD ++ mktemp + local LAST_ERR=/tmp/tmp.etRrBUuW3v + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.r0aksH8xSD No resources found + cat /tmp/tmp.etRrBUuW3v + rm /tmp/tmp.r0aksH8xSD /tmp/tmp.etRrBUuW3v + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + xargs kubectl delete ns + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.qbgzGpPvHg ++ mktemp + local LAST_ERR=/tmp/tmp.7x8JZOJklE + local exit_status=0 ++ seq 0 2 ++ mktemp + for i in '$(seq 0 2)' + set +e + kubectl get ns + local LAST_OUT=/tmp/tmp.WbWDdGsXOc + awk '{print$1}' ++ mktemp + local LAST_ERR=/tmp/tmp.P2udZxbnnu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qbgzGpPvHg + cat /tmp/tmp.7x8JZOJklE + rm /tmp/tmp.qbgzGpPvHg /tmp/tmp.7x8JZOJklE + return 0 namespace "cert-manager" deleted namespace "smart-update2-13860" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WbWDdGsXOc namespace "pxc-operator" deleted + cat /tmp/tmp.P2udZxbnnu + rm /tmp/tmp.WbWDdGsXOc /tmp/tmp.P2udZxbnnu + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.u9UCPtAARf ++ mktemp + local LAST_ERR=/tmp/tmp.zwGNnYtii5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.u9UCPtAARf namespace/pxc-operator created + cat /tmp/tmp.zwGNnYtii5 + rm /tmp/tmp.u9UCPtAARf /tmp/tmp.zwGNnYtii5 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.A43qUmRffo +++ mktemp ++ local LAST_ERR=/tmp/tmp.snwuLfSSa1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A43qUmRffo ++ cat /tmp/tmp.snwuLfSSa1 ++ rm /tmp/tmp.A43qUmRffo /tmp/tmp.snwuLfSSa1 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2202-c5e2e681-2-cluster8 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.xk4UoyeKcy ++ mktemp + local LAST_ERR=/tmp/tmp.oSAydJNjrW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2202-c5e2e681-2-cluster8 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xk4UoyeKcy Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2202-c5e2e681-2-cluster8" modified. + cat /tmp/tmp.oSAydJNjrW + rm /tmp/tmp.xk4UoyeKcy /tmp/tmp.oSAydJNjrW + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.HhLtcUR6Wh ++ mktemp + local LAST_ERR=/tmp/tmp.mBbgCvrszX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HhLtcUR6Wh customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.mBbgCvrszX + rm /tmp/tmp.HhLtcUR6Wh /tmp/tmp.mBbgCvrszX + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/deploy/cw-rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.b5hebTOmOF ++ mktemp + local LAST_ERR=/tmp/tmp.8ajPs8DvLj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.b5hebTOmOF clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.8ajPs8DvLj + rm /tmp/tmp.b5hebTOmOF /tmp/tmp.8ajPs8DvLj + return 0 + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2202-c5e2e681^' + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/deploy/cw-operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.QLzLWVHmko ++ mktemp + local LAST_ERR=/tmp/tmp.URyRutZ63k + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QLzLWVHmko deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.URyRutZ63k + rm /tmp/tmp.QLzLWVHmko /tmp/tmp.URyRutZ63k + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.cqfv88k8V1 ++ mktemp + local LAST_ERR=/tmp/tmp.zPV92HvLUp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cqfv88k8V1 pod/percona-xtradb-cluster-operator-d84c66648-5xbs8 condition met + cat /tmp/tmp.zPV92HvLUp + rm /tmp/tmp.cqfv88k8V1 /tmp/tmp.zPV92HvLUp + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.tUbwNhpm2Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.KE2m5B7jJU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tUbwNhpm2Q ++ cat /tmp/tmp.KE2m5B7jJU ++ rm /tmp/tmp.tUbwNhpm2Q /tmp/tmp.KE2m5B7jJU ++ return 0 + wait_pod percona-xtradb-cluster-operator-d84c66648-5xbs8 480 pxc-operator + local pod=percona-xtradb-cluster-operator-d84c66648-5xbs8 + local max_retry=480 + local ns=pxc-operator ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo percona-xtradb-cluster-operator-d84c66648-5xbs8 ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-d84c66648-5xbs8 condition met waiting for pod/percona-xtradb-cluster-operator-d84c66648-5xbs8 to become Ready.Ok + sleep 3 + create_namespace smart-update2-10649 + local namespace=smart-update2-10649 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces smart-update2-10649' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces smart-update2-10649 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace smart-update2-10649 + xargs kubectl delete ns + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + kubectl_bin get ns ++ mktemp + awk '{print$1}' + local LAST_OUT=/tmp/tmp.Uy36gDXql1 ++ mktemp + local LAST_OUT=/tmp/tmp.CkPjzrp75b ++ mktemp + local LAST_ERR=/tmp/tmp.dccYdOg3ka + local exit_status=0 ++ seq 0 2 ++ mktemp + for i in '$(seq 0 2)' + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.aZIilexU94 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-10649 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CkPjzrp75b + cat /tmp/tmp.dccYdOg3ka + rm /tmp/tmp.CkPjzrp75b /tmp/tmp.dccYdOg3ka + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-10649 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-10649 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.Uy36gDXql1 + cat /tmp/tmp.aZIilexU94 Error from server (NotFound): namespaces "smart-update2-10649" not found + rm /tmp/tmp.Uy36gDXql1 /tmp/tmp.aZIilexU94 + return 1 + : + wait_for_delete namespace/smart-update2-10649 + local res=namespace/smart-update2-10649 + echo -n 'waiting for namespace/smart-update2-10649 to be deleted' waiting for namespace/smart-update2-10649 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "smart-update2-10649" not found + desc 'create namespace smart-update2-10649' + set +o xtrace ----------------------------------------------------------------------------------- create namespace smart-update2-10649 ----------------------------------------------------------------------------------- + kubectl_bin create namespace smart-update2-10649 ++ mktemp + local LAST_OUT=/tmp/tmp.lLtNCOHc26 ++ mktemp + local LAST_ERR=/tmp/tmp.xq85nOPc9w + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace smart-update2-10649 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lLtNCOHc26 namespace/smart-update2-10649 created + cat /tmp/tmp.xq85nOPc9w + rm /tmp/tmp.lLtNCOHc26 /tmp/tmp.xq85nOPc9w + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.5qYojiAwB2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.CddxK6Ta0P ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5qYojiAwB2 ++ cat /tmp/tmp.CddxK6Ta0P ++ rm /tmp/tmp.5qYojiAwB2 /tmp/tmp.CddxK6Ta0P ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2202-c5e2e681-2-cluster8 --namespace=smart-update2-10649 ++ mktemp + local LAST_OUT=/tmp/tmp.qQgPObdLhK ++ mktemp + local LAST_ERR=/tmp/tmp.oNneQuUBdj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2202-c5e2e681-2-cluster8 --namespace=smart-update2-10649 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qQgPObdLhK Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2202-c5e2e681-2-cluster8" modified. + cat /tmp/tmp.oNneQuUBdj + rm /tmp/tmp.qQgPObdLhK /tmp/tmp.oNneQuUBdj + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.WpJjSqaUvW ++ mktemp + local LAST_ERR=/tmp/tmp.VQdNzJZuKy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WpJjSqaUvW secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.VQdNzJZuKy + rm /tmp/tmp.WpJjSqaUvW /tmp/tmp.VQdNzJZuKy + return 0 + deploy_version_service + desc 'install version service' + set +o xtrace ----------------------------------------------------------------------------------- install version service ----------------------------------------------------------------------------------- + kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json ++ mktemp + local LAST_OUT=/tmp/tmp.PdI6YSpDZS ++ mktemp + local LAST_ERR=/tmp/tmp.k8TKEkMJPs + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PdI6YSpDZS configmap/versions created + cat /tmp/tmp.k8TKEkMJPs + rm /tmp/tmp.PdI6YSpDZS /tmp/tmp.k8TKEkMJPs + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.n0f99CKgVW ++ mktemp + local LAST_ERR=/tmp/tmp.EhhWQzHfbT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.n0f99CKgVW deployment.apps/version-service created service/version-service created + cat /tmp/tmp.EhhWQzHfbT + rm /tmp/tmp.n0f99CKgVW /tmp/tmp.EhhWQzHfbT + return 0 + sleep 10 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.7JZ9wUacRB ++ mktemp + local LAST_ERR=/tmp/tmp.1JWRCywhKA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7JZ9wUacRB namespace/cert-manager created + cat /tmp/tmp.1JWRCywhKA + rm /tmp/tmp.7JZ9wUacRB /tmp/tmp.1JWRCywhKA + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.M1LSeDuqox ++ mktemp + local LAST_ERR=/tmp/tmp.ytopPpHo3G + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.M1LSeDuqox namespace/cert-manager labeled + cat /tmp/tmp.ytopPpHo3G + rm /tmp/tmp.M1LSeDuqox /tmp/tmp.ytopPpHo3G + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.18.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.JiPimIKAFM ++ mktemp + local LAST_ERR=/tmp/tmp.oZfKXLTAfk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.18.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JiPimIKAFM namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager-cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.oZfKXLTAfk Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.JiPimIKAFM /tmp/tmp.oZfKXLTAfk + return 0 + '[' '' == 4.10 ']' + sleep 70 ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ grep :8.0 ++ sort -V ++ tail -n3 ++ head -n1 +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++++ grep -c percona-xtradb-cluster-operator ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0EC6CfycR8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.v54dEqqjDQ +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.0EC6CfycR8 +++ cat /tmp/tmp.v54dEqqjDQ +++ rm /tmp/tmp.0EC6CfycR8 /tmp/tmp.v54dEqqjDQ +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-d84c66648-5xbs8 -n pxc-operator -- curl -s http://version-service.smart-update2-10649.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.0CIV7Gp1EP +++ mktemp ++ local LAST_ERR=/tmp/tmp.ThXIq6y41f ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-d84c66648-5xbs8 -n pxc-operator -- curl -s http://version-service.smart-update2-10649.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0CIV7Gp1EP ++ cat /tmp/tmp.ThXIq6y41f Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.0CIV7Gp1EP /tmp/tmp.ThXIq6y41f ++ return 0 + IMAGE_PXC=percona/percona-xtradb-cluster:8.0.19-10.1 + kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' ++ mktemp + local LAST_OUT=/tmp/tmp.yNyzQXkZ0m ++ mktemp + local LAST_ERR=/tmp/tmp.gds26VxkA8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yNyzQXkZ0m customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched + cat /tmp/tmp.gds26VxkA8 + rm /tmp/tmp.yNyzQXkZ0m /tmp/tmp.gds26VxkA8 + return 0 + kubectl_bin -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-10649.svc.cluster.local:11000 ++ mktemp + local LAST_OUT=/tmp/tmp.tDhnOUUBof ++ mktemp + local LAST_ERR=/tmp/tmp.mhXWOyvPmD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-10649.svc.cluster.local:11000 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tDhnOUUBof deployment.apps/percona-xtradb-cluster-operator env updated + cat /tmp/tmp.mhXWOyvPmD + rm /tmp/tmp.tDhnOUUBof /tmp/tmp.mhXWOyvPmD + return 0 + desc 'Starting telemetry testing' + set +o xtrace ----------------------------------------------------------------------------------- Starting telemetry testing ----------------------------------------------------------------------------------- + /usr/bin/sed s/version-service/version-service-cr/g /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/conf/vs.yml ++ yq 'select(.kind == "Deployment").spec.template.spec.containers[0].image' /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/conf/vs.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.4AUlTp5S2B + yq eval '(. | select(.kind == "Deployment") | .spec.template.spec.containers[0].image) = "perconalab/version-service:main-e378a19"' ++ mktemp + local LAST_ERR=/tmp/tmp.jbxovr1fNV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4AUlTp5S2B deployment.apps/version-service-cr created service/version-service-cr created + cat /tmp/tmp.jbxovr1fNV + rm /tmp/tmp.4AUlTp5S2B /tmp/tmp.jbxovr1fNV + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.TLrHZHJq6T ++ mktemp + local LAST_ERR=/tmp/tmp.CtqgutoI6F + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TLrHZHJq6T pod "version-service-798f66646-nzqcl" deleted from smart-update2-10649 namespace + cat /tmp/tmp.CtqgutoI6F + rm /tmp/tmp.TLrHZHJq6T /tmp/tmp.CtqgutoI6F + return 0 ++ echo -n percona/percona-xtradb-cluster:8.0.19-10.1 ++ sed -r 's/^.*:([0-9]+.[0-9]+).*/\1/' + IMAGE_PREFIX=8.0 + desc 'Enable telemetry on operator level' + set +o xtrace ----------------------------------------------------------------------------------- Enable telemetry on operator level ----------------------------------------------------------------------------------- + yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "false"' + kubectl_bin apply -n pxc-operator -f - + kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.xUQF5TZy3z + local LAST_OUT=/tmp/tmp.Kj1JVAHpGS ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.ci8tRwfVG3 + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.7RrbjqfLgA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + for i in '$(seq 0 2)' + set +e + kubectl apply -n pxc-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xUQF5TZy3z + cat /tmp/tmp.ci8tRwfVG3 + rm /tmp/tmp.xUQF5TZy3z /tmp/tmp.ci8tRwfVG3 + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Kj1JVAHpGS deployment.apps/percona-xtradb-cluster-operator configured + cat /tmp/tmp.7RrbjqfLgA + rm /tmp/tmp.Kj1JVAHpGS /tmp/tmp.7RrbjqfLgA + return 0 + sleep 30 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.O2VZSE00Om +++ mktemp ++ local LAST_ERR=/tmp/tmp.jrQpyZ4eI0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.O2VZSE00Om ++ cat /tmp/tmp.jrQpyZ4eI0 ++ rm /tmp/tmp.O2VZSE00Om /tmp/tmp.jrQpyZ4eI0 ++ return 0 + wait_pod percona-xtradb-cluster-operator-56c7dd9c87-wgd8g 480 pxc-operator + local pod=percona-xtradb-cluster-operator-56c7dd9c87-wgd8g + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-56c7dd9c87-wgd8g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-56c7dd9c87-wgd8g condition met waiting for pod/percona-xtradb-cluster-operator-56c7dd9c87-wgd8g to become Ready.Ok + check_telemetry_transfer http://version-service-cr.smart-update2-10649.svc.cluster.local:11000 disabled enabled + local cr_vs_uri=http://version-service-cr.smart-update2-10649.svc.cluster.local:11000 + local cr_vs_channel=disabled + local telemetry_state=enabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.LaIzxbmQpW ++ mktemp + local LAST_ERR=/tmp/tmp.7YfGf9Eetq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LaIzxbmQpW deployment.apps/pxc-client created + cat /tmp/tmp.7YfGf9Eetq + rm /tmp/tmp.LaIzxbmQpW /tmp/tmp.7YfGf9Eetq + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/secrets.yml + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.YmSf3mSs7v ++ mktemp + local LAST_ERR=/tmp/tmp.KbGPnxFIp8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YmSf3mSs7v secret/minimal-cluster created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.KbGPnxFIp8 + rm /tmp/tmp.YmSf3mSs7v /tmp/tmp.KbGPnxFIp8 + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/deploy/cr-minimal.yaml + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-10649.svc.cluster.local:11000"' + kubectl_bin apply -f - + yq eval '.spec.crVersion="9.9.9"' + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2202-c5e2e681"' + yq eval '.spec.upgradeOptions.apply="disabled"' + yq eval '.spec.logcollector.image="perconalab/percona-xtradb-cluster-operator:main-logcollector"' + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.19-10.1"' ++ mktemp + local LAST_OUT=/tmp/tmp.xT7xuOUvPk ++ mktemp + local LAST_ERR=/tmp/tmp.lvxc02t98Z + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xT7xuOUvPk perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.lvxc02t98Z + rm /tmp/tmp.xT7xuOUvPk /tmp/tmp.lvxc02t98Z + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8U07p6miOw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zHCQFvxGPA +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.8U07p6miOw +++ cat /tmp/tmp.zHCQFvxGPA +++ rm /tmp/tmp.8U07p6miOw /tmp/tmp.zHCQFvxGPA +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zAxfVxUZs0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.nBNTNd0GOa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zAxfVxUZs0 ++ cat /tmp/tmp.nBNTNd0GOa ++ rm /tmp/tmp.zAxfVxUZs0 /tmp/tmp.nBNTNd0GOa ++ return 0 + client_pod=pxc-client-59944c5bbf-gwtt2 + wait_pod pxc-client-59944c5bbf-gwtt2 + local pod=pxc-client-59944c5bbf-gwtt2 + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-gwtt2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-gwtt2 condition met waiting for pod/pxc-client-59944c5bbf-gwtt2 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mZYqwg9YoV +++ mktemp ++ local LAST_ERR=/tmp/tmp.YEgByAyJRP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mZYqwg9YoV ++ cat /tmp/tmp.YEgByAyJRP ++ rm /tmp/tmp.mZYqwg9YoV /tmp/tmp.YEgByAyJRP ++ return 0 + client_pod=pxc-client-59944c5bbf-gwtt2 + wait_pod pxc-client-59944c5bbf-gwtt2 + local pod=pxc-client-59944c5bbf-gwtt2 + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-gwtt2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-gwtt2 condition met waiting for pod/pxc-client-59944c5bbf-gwtt2 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + grep -E 'server request payload|unary call' + grep -Eo '\{.*\}' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.request.content".msg.customResourceUid)' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.start_time")' + kubectl_bin logs version-service-cr-6f74fc9d66-b6bqh ++ mktemp + local LAST_OUT=/tmp/tmp.ELX1QcO5ZX ++ mktemp + local LAST_ERR=/tmp/tmp.vWxVprprCP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-6f74fc9d66-b6bqh + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ELX1QcO5ZX + cat /tmp/tmp.vWxVprprCP + rm /tmp/tmp.ELX1QcO5ZX /tmp/tmp.vWxVprprCP + return 0 + grep -E 'server request payload|unary call' + grep -Eo '\{.*\}' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.time_ms")' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + kubectl_bin logs version-service-798f66646-dgfbw ++ mktemp + local LAST_OUT=/tmp/tmp.FnaCyUcb76 ++ mktemp + local LAST_ERR=/tmp/tmp.r1025ikKff + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-798f66646-dgfbw + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FnaCyUcb76 + cat /tmp/tmp.r1025ikKff + rm /tmp/tmp.FnaCyUcb76 /tmp/tmp.r1025ikKff + return 0 + local telemetry_log_file=enabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' disabled == disabled -a enabled == enabled ']' + desc 'operator fallback VS should have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator fallback VS should have telemetry ----------------------------------------------------------------------------------- + diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json /dev/fd/63 ++ grep -f /tmp/tmp.7JfPSw7Gpf/enabled_telemetry.version-service.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json + desc 'CR VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- CR VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.7JfPSw7Gpf/enabled_telemetry.version-service-cr.log.json ]] + local image_prefix=disabled + local telemetry_cr_log_file=enabled_telemetry.version-service-cr-disabled-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + '[' disabled == disabled-recommended -a enabled == disabled ']' + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' disabled == disabled -a enabled == disabled ']' + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.Uul3lnTtQq ++ mktemp + local LAST_ERR=/tmp/tmp.BLiou20HJY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Uul3lnTtQq perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.BLiou20HJY + rm /tmp/tmp.Uul3lnTtQq /tmp/tmp.BLiou20HJY + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.gKUlyPYwrj +++ mktemp ++ local LAST_ERR=/tmp/tmp.EIdjauVVxg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gKUlyPYwrj ++ cat /tmp/tmp.EIdjauVVxg ++ rm /tmp/tmp.gKUlyPYwrj /tmp/tmp.EIdjauVVxg ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-56c7dd9c87-wgd8g ++ mktemp + local LAST_OUT=/tmp/tmp.4lkb16WMSM ++ mktemp + local LAST_ERR=/tmp/tmp.uQEAbwiI3J + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-56c7dd9c87-wgd8g + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4lkb16WMSM pod "percona-xtradb-cluster-operator-56c7dd9c87-wgd8g" deleted from pxc-operator namespace + cat /tmp/tmp.uQEAbwiI3J + rm /tmp/tmp.4lkb16WMSM /tmp/tmp.uQEAbwiI3J + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.k9lrQNnDDP ++ mktemp + local LAST_ERR=/tmp/tmp.Pw7umuXxxB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.k9lrQNnDDP perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-10649 namespace + cat /tmp/tmp.Pw7umuXxxB + rm /tmp/tmp.k9lrQNnDDP /tmp/tmp.Pw7umuXxxB + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.aJgbsXe5TX ++ mktemp + local LAST_ERR=/tmp/tmp.ZTdz208FzI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.aJgbsXe5TX deployment.apps "pxc-client" deleted from smart-update2-10649 namespace + cat /tmp/tmp.ZTdz208FzI + rm /tmp/tmp.aJgbsXe5TX /tmp/tmp.ZTdz208FzI + return 0 + sleep 30 + desc 'Disabling telemetry on the operator level' + set +o xtrace ----------------------------------------------------------------------------------- Disabling telemetry on the operator level ----------------------------------------------------------------------------------- + kubectl_bin delete pod -l run=version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.yj2InQddX0 ++ mktemp + local LAST_ERR=/tmp/tmp.8c9Tpo4Cx3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yj2InQddX0 pod "version-service-cr-6f74fc9d66-b6bqh" deleted from smart-update2-10649 namespace + cat /tmp/tmp.8c9Tpo4Cx3 + rm /tmp/tmp.yj2InQddX0 /tmp/tmp.8c9Tpo4Cx3 + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.IEO9IstDQS ++ mktemp + local LAST_ERR=/tmp/tmp.b5bSDzpGE6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IEO9IstDQS pod "version-service-798f66646-dgfbw" deleted from smart-update2-10649 namespace + cat /tmp/tmp.b5bSDzpGE6 + rm /tmp/tmp.IEO9IstDQS /tmp/tmp.b5bSDzpGE6 + return 0 + kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' + kubectl_bin apply -n pxc-operator -f - ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.jwZT9RlOmD + local LAST_OUT=/tmp/tmp.CSgcfhVpef ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.QJGPEjZiVE + local exit_status=0 + local LAST_ERR=/tmp/tmp.T2GiPAe0vA + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n pxc-operator -f - + for i in '$(seq 0 2)' + set +e + kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jwZT9RlOmD + cat /tmp/tmp.T2GiPAe0vA + rm /tmp/tmp.jwZT9RlOmD /tmp/tmp.T2GiPAe0vA + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CSgcfhVpef deployment.apps/percona-xtradb-cluster-operator configured + cat /tmp/tmp.QJGPEjZiVE + rm /tmp/tmp.CSgcfhVpef /tmp/tmp.QJGPEjZiVE + return 0 + sleep 30 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.SFHYBQ2Yd6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FMThfPon4p ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SFHYBQ2Yd6 ++ cat /tmp/tmp.FMThfPon4p ++ rm /tmp/tmp.SFHYBQ2Yd6 /tmp/tmp.FMThfPon4p ++ return 0 + wait_pod percona-xtradb-cluster-operator-6b86564996-ftg7z 480 pxc-operator + local pod=percona-xtradb-cluster-operator-6b86564996-ftg7z + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-6b86564996-ftg7z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-6b86564996-ftg7z condition met waiting for pod/percona-xtradb-cluster-operator-6b86564996-ftg7z to become Ready.Ok + check_telemetry_transfer http://version-service-cr.smart-update2-10649.svc.cluster.local:11000 8.0-recommended disabled + local cr_vs_uri=http://version-service-cr.smart-update2-10649.svc.cluster.local:11000 + local cr_vs_channel=8.0-recommended + local telemetry_state=disabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.mCMRzFoRlK ++ mktemp + local LAST_ERR=/tmp/tmp.FFkyCuGWZ8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mCMRzFoRlK deployment.apps/pxc-client created + cat /tmp/tmp.FFkyCuGWZ8 + rm /tmp/tmp.mCMRzFoRlK /tmp/tmp.FFkyCuGWZ8 + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/secrets.yml + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.9TWKLsOF1M ++ mktemp + local LAST_ERR=/tmp/tmp.1WnvFl4ypr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9TWKLsOF1M secret/minimal-cluster unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.1WnvFl4ypr + rm /tmp/tmp.9TWKLsOF1M /tmp/tmp.1WnvFl4ypr + return 0 + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-10649.svc.cluster.local:11000"' + yq eval '.spec.upgradeOptions.apply="8.0-recommended"' + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2202-c5e2e681"' + kubectl_bin apply -f - + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.19-10.1"' + yq eval '.spec.crVersion="9.9.9"' + yq eval '.spec.logcollector.image="perconalab/percona-xtradb-cluster-operator:main-logcollector"' + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/deploy/cr-minimal.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.oidY6W11UQ ++ mktemp + local LAST_ERR=/tmp/tmp.mktbm1gMo4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oidY6W11UQ perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.mktbm1gMo4 + rm /tmp/tmp.oidY6W11UQ /tmp/tmp.mktbm1gMo4 + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TnKnItRdXO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1Cbe2928dz +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.TnKnItRdXO +++ cat /tmp/tmp.1Cbe2928dz +++ rm /tmp/tmp.TnKnItRdXO /tmp/tmp.1Cbe2928dz +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo minimal-cluster-haproxy-0 ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ojCmyAWNyf +++ mktemp ++ local LAST_ERR=/tmp/tmp.LMGnJENdYJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ojCmyAWNyf ++ cat /tmp/tmp.LMGnJENdYJ ++ rm /tmp/tmp.ojCmyAWNyf /tmp/tmp.LMGnJENdYJ ++ return 0 + client_pod=pxc-client-59944c5bbf-h92sd + wait_pod pxc-client-59944c5bbf-h92sd + local pod=pxc-client-59944c5bbf-h92sd + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-h92sd ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-h92sd condition met waiting for pod/pxc-client-59944c5bbf-h92sd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.h5VPa0btea +++ mktemp ++ local LAST_ERR=/tmp/tmp.QOZlmUtwsC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.h5VPa0btea ++ cat /tmp/tmp.QOZlmUtwsC ++ rm /tmp/tmp.h5VPa0btea /tmp/tmp.QOZlmUtwsC ++ return 0 + client_pod=pxc-client-59944c5bbf-h92sd + wait_pod pxc-client-59944c5bbf-h92sd + local pod=pxc-client-59944c5bbf-h92sd + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-h92sd ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-h92sd condition met waiting for pod/pxc-client-59944c5bbf-h92sd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.start_time")' + grep -E 'server request payload|unary call' + grep -Eo '\{.*\}' + jq 'del(."grpc.request.content".msg.customResourceUid)' + kubectl_bin logs version-service-cr-6f74fc9d66-cgzvj ++ mktemp + local LAST_OUT=/tmp/tmp.88XqJU2D0S ++ mktemp + local LAST_ERR=/tmp/tmp.eQwV32XS43 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-6f74fc9d66-cgzvj + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.88XqJU2D0S + cat /tmp/tmp.eQwV32XS43 + rm /tmp/tmp.88XqJU2D0S /tmp/tmp.eQwV32XS43 + return 0 + grep -E 'server request payload|unary call' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.start_time")' + grep -Eo '\{.*\}' + kubectl_bin logs version-service-798f66646-qh567 ++ mktemp + local LAST_OUT=/tmp/tmp.KNC8C96Qyy ++ mktemp + local LAST_ERR=/tmp/tmp.b38em6GHTQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-798f66646-qh567 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KNC8C96Qyy + cat /tmp/tmp.b38em6GHTQ + rm /tmp/tmp.KNC8C96Qyy /tmp/tmp.b38em6GHTQ + return 0 + local telemetry_log_file=disabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' 8.0-recommended == disabled -a disabled == enabled ']' + local image_prefix=8.0 + local telemetry_cr_log_file=disabled_telemetry.version-service-cr-8.0-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + '[' 8.0-recommended == 8.0-recommended -a disabled == disabled ']' + desc 'cr VS should have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- cr VS should have telemetry ----------------------------------------------------------------------------------- + diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.0-cw.log.json /dev/fd/63 ++ grep -f /tmp/tmp.7JfPSw7Gpf/disabled_telemetry.version-service-cr.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.0-cw.log.json + desc 'operator VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.7JfPSw7Gpf/disabled_telemetry.version-service.log.json ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' 8.0-recommended == disabled -a disabled == disabled ']' + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.7xGRdmUrr3 ++ mktemp + local LAST_ERR=/tmp/tmp.OQ7Q6RyeJs + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7xGRdmUrr3 perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.OQ7Q6RyeJs + rm /tmp/tmp.7xGRdmUrr3 /tmp/tmp.OQ7Q6RyeJs + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.aYXmw44JSy +++ mktemp ++ local LAST_ERR=/tmp/tmp.G2TheQIwRv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aYXmw44JSy ++ cat /tmp/tmp.G2TheQIwRv ++ rm /tmp/tmp.aYXmw44JSy /tmp/tmp.G2TheQIwRv ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-6b86564996-ftg7z ++ mktemp + local LAST_OUT=/tmp/tmp.3bCk5Kz46o ++ mktemp + local LAST_ERR=/tmp/tmp.KXMauasGyG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-6b86564996-ftg7z + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3bCk5Kz46o pod "percona-xtradb-cluster-operator-6b86564996-ftg7z" deleted from pxc-operator namespace + cat /tmp/tmp.KXMauasGyG + rm /tmp/tmp.3bCk5Kz46o /tmp/tmp.KXMauasGyG + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.u00RstzQbO ++ mktemp + local LAST_ERR=/tmp/tmp.mPYWt8zUiO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.u00RstzQbO perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-10649 namespace + cat /tmp/tmp.mPYWt8zUiO + rm /tmp/tmp.u00RstzQbO /tmp/tmp.mPYWt8zUiO + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.seJuofmneu ++ mktemp + local LAST_ERR=/tmp/tmp.bDcMotSmf2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.seJuofmneu deployment.apps "pxc-client" deleted from smart-update2-10649 namespace + cat /tmp/tmp.bDcMotSmf2 + rm /tmp/tmp.seJuofmneu /tmp/tmp.bDcMotSmf2 + return 0 + sleep 30 + kubectl_bin delete pod -l run=version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.GrZaO4TLFY ++ mktemp + local LAST_ERR=/tmp/tmp.4em3zhf6g5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GrZaO4TLFY pod "version-service-cr-6f74fc9d66-cgzvj" deleted from smart-update2-10649 namespace + cat /tmp/tmp.4em3zhf6g5 + rm /tmp/tmp.GrZaO4TLFY /tmp/tmp.4em3zhf6g5 + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.sCHfSgko3t ++ mktemp + local LAST_ERR=/tmp/tmp.nlbAYt5QjQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sCHfSgko3t pod "version-service-798f66646-qh567" deleted from smart-update2-10649 namespace + cat /tmp/tmp.nlbAYt5QjQ + rm /tmp/tmp.sCHfSgko3t /tmp/tmp.nlbAYt5QjQ + return 0 + check_telemetry_transfer http://version-service-cr.smart-update2-10649.svc.cluster.local:11000 disabled disabled + local cr_vs_uri=http://version-service-cr.smart-update2-10649.svc.cluster.local:11000 + local cr_vs_channel=disabled + local telemetry_state=disabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.7wKSQisYHZ ++ mktemp + local LAST_ERR=/tmp/tmp.2KnZzLb5zi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7wKSQisYHZ deployment.apps/pxc-client created + cat /tmp/tmp.2KnZzLb5zi + rm /tmp/tmp.7wKSQisYHZ /tmp/tmp.2KnZzLb5zi + return 0 + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/secrets.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.hcnnk6pgdx ++ mktemp + local LAST_ERR=/tmp/tmp.0VuAArlDo5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hcnnk6pgdx secret/minimal-cluster unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.0VuAArlDo5 + rm /tmp/tmp.hcnnk6pgdx /tmp/tmp.0VuAArlDo5 + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/deploy/cr-minimal.yaml + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-10649.svc.cluster.local:11000"' + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2202-c5e2e681"' + yq eval '.spec.upgradeOptions.apply="disabled"' + yq eval '.spec.crVersion="9.9.9"' + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.19-10.1"' + kubectl_bin apply -f - + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + yq eval '.spec.logcollector.image="perconalab/percona-xtradb-cluster-operator:main-logcollector"' ++ mktemp + local LAST_OUT=/tmp/tmp.PIajVpeIuU ++ mktemp + local LAST_ERR=/tmp/tmp.BhOCWYg8LX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PIajVpeIuU perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.BhOCWYg8LX + rm /tmp/tmp.PIajVpeIuU /tmp/tmp.BhOCWYg8LX + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Z3UOidrePC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.l3nkEedxkj +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Z3UOidrePC +++ cat /tmp/tmp.l3nkEedxkj +++ rm /tmp/tmp.Z3UOidrePC /tmp/tmp.l3nkEedxkj +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo minimal-cluster-haproxy-0 + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7kPVtLFU3S +++ mktemp ++ local LAST_ERR=/tmp/tmp.PgsOnJl6ML ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7kPVtLFU3S ++ cat /tmp/tmp.PgsOnJl6ML ++ rm /tmp/tmp.7kPVtLFU3S /tmp/tmp.PgsOnJl6ML ++ return 0 + client_pod=pxc-client-59944c5bbf-fwqxc + wait_pod pxc-client-59944c5bbf-fwqxc + local pod=pxc-client-59944c5bbf-fwqxc + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-fwqxc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-fwqxc condition met waiting for pod/pxc-client-59944c5bbf-fwqxc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bmXxixPRDH +++ mktemp ++ local LAST_ERR=/tmp/tmp.fK5oSOsbqq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bmXxixPRDH ++ cat /tmp/tmp.fK5oSOsbqq ++ rm /tmp/tmp.bmXxixPRDH /tmp/tmp.fK5oSOsbqq ++ return 0 + client_pod=pxc-client-59944c5bbf-fwqxc + wait_pod pxc-client-59944c5bbf-fwqxc + local pod=pxc-client-59944c5bbf-fwqxc + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-fwqxc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-fwqxc condition met waiting for pod/pxc-client-59944c5bbf-fwqxc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument. command terminated with exit code 5 + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.kubeVersion)' + grep -Eo '\{.*\}' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + grep -E 'server request payload|unary call' + kubectl_bin logs version-service-cr-6f74fc9d66-vb89w ++ mktemp + local LAST_OUT=/tmp/tmp.IWHKaONhg1 ++ mktemp + local LAST_ERR=/tmp/tmp.u7AEd9EU5d + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-6f74fc9d66-vb89w + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IWHKaONhg1 + cat /tmp/tmp.u7AEd9EU5d + rm /tmp/tmp.IWHKaONhg1 /tmp/tmp.u7AEd9EU5d + return 0 + grep -E 'server request payload|unary call' + grep -Eo '\{.*\}' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.time_ms")' + kubectl_bin logs version-service-798f66646-mq74j ++ mktemp + local LAST_OUT=/tmp/tmp.iaTiZWvnTD ++ mktemp + local LAST_ERR=/tmp/tmp.dYZ7jnYz66 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-798f66646-mq74j + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iaTiZWvnTD + cat /tmp/tmp.dYZ7jnYz66 + rm /tmp/tmp.iaTiZWvnTD /tmp/tmp.dYZ7jnYz66 + return 0 + local telemetry_log_file=disabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' disabled == disabled -a disabled == enabled ']' + local image_prefix=disabled + local telemetry_cr_log_file=disabled_telemetry.version-service-cr-disabled-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + '[' disabled == disabled-recommended -a disabled == disabled ']' + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' disabled == disabled -a disabled == disabled ']' + desc 'CR VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- CR VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.7JfPSw7Gpf/disabled_telemetry.version-service-cr.log.json ]] + desc 'operator VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.7JfPSw7Gpf/disabled_telemetry.version-service.log.json ]] + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.lu4EJkIgck ++ mktemp + local LAST_ERR=/tmp/tmp.U71nXp8JSg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lu4EJkIgck perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.U71nXp8JSg + rm /tmp/tmp.lu4EJkIgck /tmp/tmp.U71nXp8JSg + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.NmLCn4bJSX +++ mktemp ++ local LAST_ERR=/tmp/tmp.dgFtSjKh35 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NmLCn4bJSX ++ cat /tmp/tmp.dgFtSjKh35 ++ rm /tmp/tmp.NmLCn4bJSX /tmp/tmp.dgFtSjKh35 ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-6b86564996-r4jr4 ++ mktemp + local LAST_OUT=/tmp/tmp.7bRUv35m0L ++ mktemp + local LAST_ERR=/tmp/tmp.TiYSDbNUpL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-6b86564996-r4jr4 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7bRUv35m0L pod "percona-xtradb-cluster-operator-6b86564996-r4jr4" deleted from pxc-operator namespace + cat /tmp/tmp.TiYSDbNUpL + rm /tmp/tmp.7bRUv35m0L /tmp/tmp.TiYSDbNUpL + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.gbj3OxmSZB ++ mktemp + local LAST_ERR=/tmp/tmp.yKCWu4QVDE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gbj3OxmSZB perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-10649 namespace + cat /tmp/tmp.yKCWu4QVDE + rm /tmp/tmp.gbj3OxmSZB /tmp/tmp.yKCWu4QVDE + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.hHMboUrNo2 ++ mktemp + local LAST_ERR=/tmp/tmp.c55k907l6O + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hHMboUrNo2 deployment.apps "pxc-client" deleted from smart-update2-10649 namespace + cat /tmp/tmp.c55k907l6O + rm /tmp/tmp.hHMboUrNo2 /tmp/tmp.c55k907l6O + return 0 + sleep 30 + kubectl_bin delete deployment version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.XhucUbbN53 ++ mktemp + local LAST_ERR=/tmp/tmp.gOTqrgPxuO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deployment version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XhucUbbN53 deployment.apps "version-service-cr" deleted from smart-update2-10649 namespace + cat /tmp/tmp.gOTqrgPxuO + rm /tmp/tmp.XhucUbbN53 /tmp/tmp.gOTqrgPxuO + return 0 + desc 'Telemetry testing finished' + set +o xtrace ----------------------------------------------------------------------------------- Telemetry testing finished ----------------------------------------------------------------------------------- + desc 'PXC cluster with version service offline' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster with version service offline ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/conf/smart-update-version-service-unreachable.yml /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-2202-c5e2e681"' /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + spinup_pxc smart-update /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.1zhX4EOeDw ++ mktemp + local LAST_ERR=/tmp/tmp.SJqH7BsUWA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1zhX4EOeDw secret/my-cluster-secrets created secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.SJqH7BsUWA + rm /tmp/tmp.1zhX4EOeDw /tmp/tmp.SJqH7BsUWA + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + local LAST_OUT=/tmp/tmp.NITk1hP2N6 + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2202-c5e2e681#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_ERR=/tmp/tmp.ZzbOr6JLUw + local exit_status=0 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-10649~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NITk1hP2N6 deployment.apps/pxc-client created + cat /tmp/tmp.ZzbOr6JLUw + rm /tmp/tmp.NITk1hP2N6 /tmp/tmp.ZzbOr6JLUw + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + '[' -z '' ']' + cat_config /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + kubectl_bin apply -f - + cat /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + local LAST_OUT=/tmp/tmp.dlTIroohqL + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-10649~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + local LAST_ERR=/tmp/tmp.ZC48aXCiuU + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2202-c5e2e681#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dlTIroohqL perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.ZC48aXCiuU + rm /tmp/tmp.dlTIroohqL /tmp/tmp.ZC48aXCiuU + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.iqH9Wh8xQb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ElNA7annbV +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.iqH9Wh8xQb +++ cat /tmp/tmp.ElNA7annbV +++ rm /tmp/tmp.iqH9Wh8xQb /tmp/tmp.ElNA7annbV +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-10649 ++ mktemp + local LAST_OUT=/tmp/tmp.rQjIo00ccg ++ mktemp + local LAST_ERR=/tmp/tmp.OPdVIBgsP0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-10649 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-10649 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-10649 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.rQjIo00ccg + cat /tmp/tmp.OPdVIBgsP0 error: no matching resources found + rm /tmp/tmp.rQjIo00ccg /tmp/tmp.OPdVIBgsP0 + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace Error from server (NotFound): pods "smart-update-haproxy-0" not found waiting for pod/smart-update-haproxy-0 to become Ready...................................Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2wNhg3cxHr ++ base64 --decode +++ mktemp ++ local LAST_ERR=/tmp/tmp.3xv7F4cNI0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2wNhg3cxHr ++ cat /tmp/tmp.3xv7F4cNI0 ++ rm /tmp/tmp.2wNhg3cxHr /tmp/tmp.3xv7F4cNI0 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Maop80G3m9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.eVWlR8DCnK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Maop80G3m9 ++ cat /tmp/tmp.eVWlR8DCnK ++ rm /tmp/tmp.Maop80G3m9 /tmp/tmp.eVWlR8DCnK ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CaCaqWPkhs +++ mktemp ++ local LAST_ERR=/tmp/tmp.8eJdnCHP6I ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CaCaqWPkhs ++ cat /tmp/tmp.8eJdnCHP6I ++ rm /tmp/tmp.CaCaqWPkhs /tmp/tmp.8eJdnCHP6I ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5qfloZ420a +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZayPDvkd3h ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5qfloZ420a ++ cat /tmp/tmp.ZayPDvkd3h ++ rm /tmp/tmp.5qfloZ420a /tmp/tmp.ZayPDvkd3h ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-7d45b68d8f-dpgtl ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XZg54NYXSP +++ mktemp ++ local LAST_ERR=/tmp/tmp.7PrbSE8UqW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XZg54NYXSP ++ cat /tmp/tmp.7PrbSE8UqW ++ rm /tmp/tmp.XZg54NYXSP /tmp/tmp.7PrbSE8UqW ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-7d45b68d8f-dpgtl ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JGS8cDExlM +++ mktemp ++ local LAST_ERR=/tmp/tmp.INlyTkinX5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JGS8cDExlM ++ cat /tmp/tmp.INlyTkinX5 ++ rm /tmp/tmp.JGS8cDExlM /tmp/tmp.INlyTkinX5 ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-7d45b68d8f-dpgtl + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NhB1BNGf1y +++ mktemp ++ local LAST_ERR=/tmp/tmp.CK979GmPXP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NhB1BNGf1y ++ cat /tmp/tmp.CK979GmPXP Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.NhB1BNGf1y /tmp/tmp.CK979GmPXP ++ return 0 + '[' '' ']' + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=300 + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KZVj4DOk2F +++ mktemp ++ local LAST_ERR=/tmp/tmp.xqMyo3MsKZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KZVj4DOk2F ++ cat /tmp/tmp.xqMyo3MsKZ ++ rm /tmp/tmp.KZVj4DOk2F /tmp/tmp.xqMyo3MsKZ ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BAVs6pdjHG +++ mktemp ++ local LAST_ERR=/tmp/tmp.5Fl3Jg3U2x ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BAVs6pdjHG ++ cat /tmp/tmp.5Fl3Jg3U2x ++ rm /tmp/tmp.BAVs6pdjHG /tmp/tmp.5Fl3Jg3U2x ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Nv6CmY2XIO ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.DUNdigrtWR +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Nv6CmY2XIO +++++ cat /tmp/tmp.DUNdigrtWR +++++ rm /tmp/tmp.Nv6CmY2XIO /tmp/tmp.DUNdigrtWR +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.32r7NvrweS +++ mktemp ++ local LAST_ERR=/tmp/tmp.NCes9b63am ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.32r7NvrweS ++ cat /tmp/tmp.NCes9b63am ++ rm /tmp/tmp.32r7NvrweS /tmp/tmp.NCes9b63am ++ return 0 + [[ 2 == \2 ]] + echo ++ kubectl_bin get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.78v5PQ6gTZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.na7oC0ZFqY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.78v5PQ6gTZ ++ cat /tmp/tmp.na7oC0ZFqY ++ rm /tmp/tmp.78v5PQ6gTZ /tmp/tmp.na7oC0ZFqY ++ return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 != \p\e\r\c\o\n\a\/\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\:\8\.\0\.\1\9\-\1\0\.\1 ]] + desc 'PXC cluster update with recommended image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with recommended image by version service ----------------------------------------------------------------------------------- + vs_image=recommended ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Y1frNLotwR ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bzz5A4Y6FG +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Y1frNLotwR +++ cat /tmp/tmp.bzz5A4Y6FG +++ rm /tmp/tmp.Y1frNLotwR /tmp/tmp.bzz5A4Y6FG +++ return 0 ++ client_pod=pxc-client-7d45b68d8f-dpgtl ++ wait_pod pxc-client-7d45b68d8f-dpgtl ++ local pod=pxc-client-7d45b68d8f-dpgtl ++ local max_retry=480 ++ local ns= +++ echo pxc-client-7d45b68d8f-dpgtl +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.El0fdiALrX ++ mktemp + local LAST_ERR=/tmp/tmp.9c7HxssbJV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.El0fdiALrX perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.9c7HxssbJV + rm /tmp/tmp.El0fdiALrX /tmp/tmp.9c7HxssbJV + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=300 + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tncfrZpLwa +++ mktemp ++ local LAST_ERR=/tmp/tmp.5Wp1nk9Awa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tncfrZpLwa ++ cat /tmp/tmp.5Wp1nk9Awa ++ rm /tmp/tmp.tncfrZpLwa /tmp/tmp.5Wp1nk9Awa ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HB1zmBpHcY +++ mktemp ++ local LAST_ERR=/tmp/tmp.o72YtnuAts ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HB1zmBpHcY ++ cat /tmp/tmp.o72YtnuAts ++ rm /tmp/tmp.HB1zmBpHcY /tmp/tmp.o72YtnuAts ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.SUDfZpT6nv ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.j6tti6SZfY +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.SUDfZpT6nv +++++ cat /tmp/tmp.j6tti6SZfY +++++ rm /tmp/tmp.SUDfZpT6nv /tmp/tmp.j6tti6SZfY +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4cp5UNPKNk +++ mktemp ++ local LAST_ERR=/tmp/tmp.updReNKDrD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4cp5UNPKNk ++ cat /tmp/tmp.updReNKDrD ++ rm /tmp/tmp.4cp5UNPKNk /tmp/tmp.updReNKDrD ++ return 0 + [[ 2 == \2 ]] + echo ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GeXQc4uBEq +++ mktemp ++ local LAST_ERR=/tmp/tmp.Dz2F1WvIg8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GeXQc4uBEq ++ cat /tmp/tmp.Dz2F1WvIg8 ++ rm /tmp/tmp.GeXQc4uBEq /tmp/tmp.Dz2F1WvIg8 ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-7d45b68d8f-dpgtl + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.39RgJmqmgm +++ mktemp ++ local LAST_ERR=/tmp/tmp.nIBXTyuBmM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.39RgJmqmgm ++ cat /tmp/tmp.nIBXTyuBmM ++ rm /tmp/tmp.39RgJmqmgm /tmp/tmp.nIBXTyuBmM ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QfuCU0qyLc +++ mktemp ++ local LAST_ERR=/tmp/tmp.f2MazmPuNv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QfuCU0qyLc ++ cat /tmp/tmp.f2MazmPuNv ++ rm /tmp/tmp.QfuCU0qyLc /tmp/tmp.f2MazmPuNv ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + kubectl_bin delete -f /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.XrBArSOuYk ++ mktemp + local LAST_ERR=/tmp/tmp.CCmevvOS1y + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XrBArSOuYk perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-10649 namespace + cat /tmp/tmp.CCmevvOS1y + rm /tmp/tmp.XrBArSOuYk /tmp/tmp.CCmevvOS1y + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.kdGUXrKNNa ++ mktemp + local LAST_ERR=/tmp/tmp.im1ikNJNMT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kdGUXrKNNa persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-10649 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-10649 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-10649 namespace + cat /tmp/tmp.im1ikNJNMT + rm /tmp/tmp.kdGUXrKNNa /tmp/tmp.im1ikNJNMT + return 0 + desc 'PXC cluster update with the latest image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with the latest image by version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.kSgmkeS7jV ++ mktemp + local LAST_ERR=/tmp/tmp.TFbMSt0k4O + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kSgmkeS7jV secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.TFbMSt0k4O + rm /tmp/tmp.kSgmkeS7jV /tmp/tmp.TFbMSt0k4O + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-10649~ + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + local LAST_OUT=/tmp/tmp.LyB6XOUuzi ++ mktemp + local LAST_ERR=/tmp/tmp.8gyfLasl6g + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2202-c5e2e681#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LyB6XOUuzi deployment.apps/pxc-client unchanged + cat /tmp/tmp.8gyfLasl6g + rm /tmp/tmp.LyB6XOUuzi /tmp/tmp.8gyfLasl6g + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + cat /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_OUT=/tmp/tmp.Rp6qUNHtuw + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-10649~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2202-c5e2e681#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + local LAST_ERR=/tmp/tmp.5dKpLIHlFC + local exit_status=0 + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Rp6qUNHtuw perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.5dKpLIHlFC + rm /tmp/tmp.Rp6qUNHtuw /tmp/tmp.5dKpLIHlFC + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kxYq3Aynz6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ozMpwjmFvh +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.kxYq3Aynz6 +++ cat /tmp/tmp.ozMpwjmFvh +++ rm /tmp/tmp.kxYq3Aynz6 /tmp/tmp.ozMpwjmFvh +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-10649 ++ mktemp + local LAST_OUT=/tmp/tmp.lv7PN0LgTr ++ mktemp + local LAST_ERR=/tmp/tmp.cqdfOG9v4g + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-10649 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-10649 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-10649 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.lv7PN0LgTr + cat /tmp/tmp.cqdfOG9v4g error: no matching resources found + rm /tmp/tmp.lv7PN0LgTr /tmp/tmp.cqdfOG9v4g + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met waiting for pod/smart-update-haproxy-0 to become Ready.Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-0 + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.wALyUS8R70 +++ mktemp ++ local LAST_ERR=/tmp/tmp.UTTwqotrDs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wALyUS8R70 ++ cat /tmp/tmp.UTTwqotrDs ++ rm /tmp/tmp.wALyUS8R70 /tmp/tmp.UTTwqotrDs ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8PyE0Eu03H +++ mktemp ++ local LAST_ERR=/tmp/tmp.2vGxiTTGtJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8PyE0Eu03H ++ cat /tmp/tmp.2vGxiTTGtJ ++ rm /tmp/tmp.8PyE0Eu03H /tmp/tmp.2vGxiTTGtJ ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v1SBftsUTM +++ mktemp ++ local LAST_ERR=/tmp/tmp.6YURI6VcT9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.v1SBftsUTM ++ cat /tmp/tmp.6YURI6VcT9 ++ rm /tmp/tmp.v1SBftsUTM /tmp/tmp.6YURI6VcT9 ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JLmBrZtpgY +++ mktemp ++ local LAST_ERR=/tmp/tmp.zNAFQBtIiK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JLmBrZtpgY ++ cat /tmp/tmp.zNAFQBtIiK ++ rm /tmp/tmp.JLmBrZtpgY /tmp/tmp.zNAFQBtIiK ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LBfVXzkDhB +++ mktemp ++ local LAST_ERR=/tmp/tmp.zb7a5tPfvx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LBfVXzkDhB ++ cat /tmp/tmp.zb7a5tPfvx ++ rm /tmp/tmp.LBfVXzkDhB /tmp/tmp.zb7a5tPfvx ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MMyGTqWl15 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FAdeTc3LBw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MMyGTqWl15 ++ cat /tmp/tmp.FAdeTc3LBw ++ rm /tmp/tmp.MMyGTqWl15 /tmp/tmp.FAdeTc3LBw ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ egrep -o 'early-plugin-load=keyring_\w+.so' ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AB6asQwtCA +++ mktemp ++ local LAST_ERR=/tmp/tmp.UNueGi3kto ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AB6asQwtCA ++ cat /tmp/tmp.UNueGi3kto Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.AB6asQwtCA /tmp/tmp.UNueGi3kto ++ return 0 + '[' '' ']' + vs_image=latest ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qNGgZWYXQy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ndKfUJPxGV +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.qNGgZWYXQy +++ cat /tmp/tmp.ndKfUJPxGV +++ rm /tmp/tmp.qNGgZWYXQy /tmp/tmp.ndKfUJPxGV +++ return 0 ++ client_pod=pxc-client-7d45b68d8f-dpgtl ++ wait_pod pxc-client-7d45b68d8f-dpgtl ++ local pod=pxc-client-7d45b68d8f-dpgtl ++ local max_retry=480 ++ local ns= +++ echo pxc-client-7d45b68d8f-dpgtl +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.1cGpx5QbYj ++ mktemp + local LAST_ERR=/tmp/tmp.V3UVpch0n5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1cGpx5QbYj perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.V3UVpch0n5 + rm /tmp/tmp.1cGpx5QbYj /tmp/tmp.V3UVpch0n5 + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=300 + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jQcJTk4GfI +++ mktemp ++ local LAST_ERR=/tmp/tmp.3XonqO0sgN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jQcJTk4GfI ++ cat /tmp/tmp.3XonqO0sgN ++ rm /tmp/tmp.jQcJTk4GfI /tmp/tmp.3XonqO0sgN ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d6EcXZFPEK +++ mktemp ++ local LAST_ERR=/tmp/tmp.IrqCtSfotv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.d6EcXZFPEK ++ cat /tmp/tmp.IrqCtSfotv ++ rm /tmp/tmp.d6EcXZFPEK /tmp/tmp.IrqCtSfotv ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.ZB1a9v22Eh ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.3gm2GBMNAH +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.ZB1a9v22Eh +++++ cat /tmp/tmp.3gm2GBMNAH +++++ rm /tmp/tmp.ZB1a9v22Eh /tmp/tmp.3gm2GBMNAH +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6IDqR6RTeL +++ mktemp ++ local LAST_ERR=/tmp/tmp.suVpEv6caP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6IDqR6RTeL ++ cat /tmp/tmp.suVpEv6caP ++ rm /tmp/tmp.6IDqR6RTeL /tmp/tmp.suVpEv6caP ++ return 0 + [[ 2 == \2 ]] + echo ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FErGLncvDF +++ mktemp ++ local LAST_ERR=/tmp/tmp.5CIGtq1e40 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FErGLncvDF ++ cat /tmp/tmp.5CIGtq1e40 ++ rm /tmp/tmp.FErGLncvDF /tmp/tmp.5CIGtq1e40 ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4KGnQDLBet +++ mktemp ++ local LAST_ERR=/tmp/tmp.SQXzeepUps ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4KGnQDLBet ++ cat /tmp/tmp.SQXzeepUps ++ rm /tmp/tmp.4KGnQDLBet /tmp/tmp.SQXzeepUps ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Nu989qDuZs +++ mktemp ++ local LAST_ERR=/tmp/tmp.WP5zLgnn8w ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Nu989qDuZs ++ cat /tmp/tmp.WP5zLgnn8w ++ rm /tmp/tmp.Nu989qDuZs /tmp/tmp.WP5zLgnn8w ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + kubectl_bin delete -f /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.9eAwIx5ozl ++ mktemp + local LAST_ERR=/tmp/tmp.fjtsIh8y1f + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9eAwIx5ozl perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-10649 namespace + cat /tmp/tmp.fjtsIh8y1f + rm /tmp/tmp.9eAwIx5ozl /tmp/tmp.fjtsIh8y1f + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.H63UcAwWrN ++ mktemp + local LAST_ERR=/tmp/tmp.CGb804C2FU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.H63UcAwWrN persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-10649 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-10649 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-10649 namespace + cat /tmp/tmp.CGb804C2FU + rm /tmp/tmp.H63UcAwWrN /tmp/tmp.CGb804C2FU + return 0 + desc 'PXC cluster update with explicitly specified image inside version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with explicitly specified image inside version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.fvDAHCZ8yq ++ mktemp + local LAST_ERR=/tmp/tmp.n8l55DW25Z + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fvDAHCZ8yq secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.n8l55DW25Z + rm /tmp/tmp.fvDAHCZ8yq /tmp/tmp.n8l55DW25Z + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2202-c5e2e681#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_OUT=/tmp/tmp.s4uzFS2PH7 ++ mktemp + local LAST_ERR=/tmp/tmp.qOt2HgjedG + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-10649~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.s4uzFS2PH7 deployment.apps/pxc-client unchanged + cat /tmp/tmp.qOt2HgjedG + rm /tmp/tmp.s4uzFS2PH7 /tmp/tmp.qOt2HgjedG + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + '[' -z '' ']' + cat_config /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + cat /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2202-c5e2e681#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-10649~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.WVJkAS7VMy + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + local LAST_ERR=/tmp/tmp.kHSGAYfqta + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WVJkAS7VMy perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.kHSGAYfqta + rm /tmp/tmp.WVJkAS7VMy /tmp/tmp.kHSGAYfqta + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4p5tnsgA6y ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0OwTqctXvt +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.4p5tnsgA6y +++ cat /tmp/tmp.0OwTqctXvt +++ rm /tmp/tmp.4p5tnsgA6y /tmp/tmp.0OwTqctXvt +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-10649 ++ mktemp + local LAST_OUT=/tmp/tmp.cmDyooDocQ ++ mktemp + local LAST_ERR=/tmp/tmp.kd8d5R3stH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-10649 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-10649 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-10649 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.cmDyooDocQ + cat /tmp/tmp.kd8d5R3stH error: no matching resources found + rm /tmp/tmp.cmDyooDocQ /tmp/tmp.kd8d5R3stH + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met waiting for pod/smart-update-haproxy-0 to become Ready.Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo smart-update-pxc-0 + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met waiting for pod/smart-update-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met waiting for pod/smart-update-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met waiting for pod/smart-update-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.ndxDMftszr +++ mktemp ++ local LAST_ERR=/tmp/tmp.lMR16MnqaR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ndxDMftszr ++ cat /tmp/tmp.lMR16MnqaR ++ rm /tmp/tmp.ndxDMftszr /tmp/tmp.lMR16MnqaR ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZUnbFyd459 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZIp4aGLPDs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZUnbFyd459 ++ cat /tmp/tmp.ZIp4aGLPDs ++ rm /tmp/tmp.ZUnbFyd459 /tmp/tmp.ZIp4aGLPDs ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-7d45b68d8f-dpgtl ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QIsYglbo4A +++ mktemp ++ local LAST_ERR=/tmp/tmp.SHNzwkyMJX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QIsYglbo4A ++ cat /tmp/tmp.SHNzwkyMJX ++ rm /tmp/tmp.QIsYglbo4A /tmp/tmp.SHNzwkyMJX ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qdZSEN7j4E +++ mktemp ++ local LAST_ERR=/tmp/tmp.qGUyVCh907 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qdZSEN7j4E ++ cat /tmp/tmp.qGUyVCh907 ++ rm /tmp/tmp.qdZSEN7j4E /tmp/tmp.qGUyVCh907 ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IWoiHr7uPv +++ mktemp ++ local LAST_ERR=/tmp/tmp.tiEFWu31vp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IWoiHr7uPv ++ cat /tmp/tmp.tiEFWu31vp ++ rm /tmp/tmp.IWoiHr7uPv /tmp/tmp.tiEFWu31vp ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fj0ZyaKJiG +++ mktemp ++ local LAST_ERR=/tmp/tmp.5uD8oeleex ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fj0ZyaKJiG ++ cat /tmp/tmp.5uD8oeleex ++ rm /tmp/tmp.fj0ZyaKJiG /tmp/tmp.5uD8oeleex ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xO4CCHcBk4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.XP8u0VsZVS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xO4CCHcBk4 ++ cat /tmp/tmp.XP8u0VsZVS Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.xO4CCHcBk4 /tmp/tmp.XP8u0VsZVS ++ return 0 + '[' '' ']' ++ jq -r '.versions[].matrix.pxc[].imagePath' +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++ head -n1 ++ grep :8.0 ++ tail -n2 ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ sort -V ++++ grep -c percona-xtradb-cluster-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gyeiGlJG5f ++++ mktemp +++ local LAST_ERR=/tmp/tmp.F4ftkU26Vj +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.gyeiGlJG5f +++ cat /tmp/tmp.F4ftkU26Vj +++ rm /tmp/tmp.gyeiGlJG5f /tmp/tmp.F4ftkU26Vj +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-6b86564996-69q2t -n pxc-operator -- curl -s http://version-service.smart-update2-10649.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.SRaWrdXr9g +++ mktemp ++ local LAST_ERR=/tmp/tmp.E9tAxunONN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-6b86564996-69q2t -n pxc-operator -- curl -s http://version-service.smart-update2-10649.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SRaWrdXr9g ++ cat /tmp/tmp.E9tAxunONN Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.SRaWrdXr9g /tmp/tmp.E9tAxunONN ++ return 0 + vs_image=percona/percona-xtradb-cluster:8.0.19-10.1 ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LDaPR4aeoe ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gGDPchL1dt +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.LDaPR4aeoe +++ cat /tmp/tmp.gGDPchL1dt +++ rm /tmp/tmp.LDaPR4aeoe /tmp/tmp.gGDPchL1dt +++ return 0 ++ client_pod=pxc-client-7d45b68d8f-dpgtl ++ wait_pod pxc-client-7d45b68d8f-dpgtl ++ local pod=pxc-client-7d45b68d8f-dpgtl ++ local max_retry=480 ++ local ns= +++ egrep '^(pxc|proxysql)$' +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ echo pxc-client-7d45b68d8f-dpgtl ++ local container= ++ set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.19-10.1","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.9pwmcArNeJ ++ mktemp + local LAST_ERR=/tmp/tmp.6z8VcDhzsN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.19-10.1","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9pwmcArNeJ perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.6z8VcDhzsN + rm /tmp/tmp.9pwmcArNeJ /tmp/tmp.6z8VcDhzsN + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.19-10.1 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.19-10.1 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=300 + sleep 7 + echo -n 'waiting for pxc/smart-update to be ready' waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K9qwMrpVAj +++ mktemp ++ local LAST_ERR=/tmp/tmp.9b44Fou6TE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.K9qwMrpVAj ++ cat /tmp/tmp.9b44Fou6TE ++ rm /tmp/tmp.K9qwMrpVAj /tmp/tmp.9b44Fou6TE ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zPVyqND7L4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.w1rydANvwT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zPVyqND7L4 ++ cat /tmp/tmp.w1rydANvwT ++ rm /tmp/tmp.zPVyqND7L4 /tmp/tmp.w1rydANvwT ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.119S55tdd9 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.g3AZDpcnEG +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.119S55tdd9 +++++ cat /tmp/tmp.g3AZDpcnEG +++++ rm /tmp/tmp.119S55tdd9 /tmp/tmp.g3AZDpcnEG +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.A4ECS8LWmE +++ mktemp ++ local LAST_ERR=/tmp/tmp.YUuJWOdBll ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A4ECS8LWmE ++ cat /tmp/tmp.YUuJWOdBll ++ rm /tmp/tmp.A4ECS8LWmE /tmp/tmp.YUuJWOdBll ++ return 0 + [[ 2 == \2 ]] + echo ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NajPhfMy3I +++ mktemp ++ local LAST_ERR=/tmp/tmp.fkb5LQ7jcC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NajPhfMy3I ++ cat /tmp/tmp.fkb5LQ7jcC ++ rm /tmp/tmp.NajPhfMy3I /tmp/tmp.fkb5LQ7jcC ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZGBbOa3haO +++ mktemp ++ local LAST_ERR=/tmp/tmp.GtCFgiucOl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZGBbOa3haO ++ cat /tmp/tmp.GtCFgiucOl ++ rm /tmp/tmp.ZGBbOa3haO /tmp/tmp.GtCFgiucOl ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.4 ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1-80.sql ]] + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.D24z1Y2Efd +++ mktemp ++ local LAST_ERR=/tmp/tmp.PQ7QdF7R3C ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.D24z1Y2Efd ++ cat /tmp/tmp.PQ7QdF7R3C ++ rm /tmp/tmp.D24z1Y2Efd /tmp/tmp.PQ7QdF7R3C ++ return 0 + client_pod=pxc-client-7d45b68d8f-dpgtl + wait_pod pxc-client-7d45b68d8f-dpgtl + local pod=pxc-client-7d45b68d8f-dpgtl + local max_retry=480 + local ns= ++ echo pxc-client-7d45b68d8f-dpgtl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-7d45b68d8f-dpgtl condition met waiting for pod/pxc-client-7d45b68d8f-dpgtl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.7JfPSw7Gpf/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.7JfPSw7Gpf/select-1.sql + kubectl_bin delete -f /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Mvox2fAsxw ++ mktemp + local LAST_ERR=/tmp/tmp.6tMkXX5u8h + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.7JfPSw7Gpf/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Mvox2fAsxw perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-10649 namespace + cat /tmp/tmp.6tMkXX5u8h + rm /tmp/tmp.Mvox2fAsxw /tmp/tmp.6tMkXX5u8h + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.hKFJl5gQfy ++ mktemp + local LAST_ERR=/tmp/tmp.qCZn3zpjnX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hKFJl5gQfy persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-10649 namespace persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-10649 namespace persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-10649 namespace + cat /tmp/tmp.qCZn3zpjnX + rm /tmp/tmp.hKFJl5gQfy /tmp/tmp.qCZn3zpjnX + return 0 + desc cleanup + set +o xtrace ----------------------------------------------------------------------------------- cleanup ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.7wFZRmN1uK ++ mktemp + local LAST_ERR=/tmp/tmp.BA2etfxBpY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2202/e2e-tests/smart-update2/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7wFZRmN1uK deployment.apps "version-service" deleted from smart-update2-10649 namespace service "version-service" deleted from smart-update2-10649 namespace + cat /tmp/tmp.BA2etfxBpY + rm /tmp/tmp.7wFZRmN1uK /tmp/tmp.BA2etfxBpY + return 0 + destroy smart-update2-10649 + local namespace=smart-update2-10649 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + sort -u + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v level=info ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ + grep -v 'the object has been modified' + tee /tmp/tmp.7JfPSw7Gpf/operator.log +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator + grep -v 'get backup status: Job.batch' +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.pei3r5GQJb +++ mktemp ++ local LAST_ERR=/tmp/tmp.kmoTRz4IO3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pei3r5GQJb ++ cat /tmp/tmp.kmoTRz4IO3 ++ rm /tmp/tmp.pei3r5GQJb /tmp/tmp.kmoTRz4IO3 ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-6b86564996-69q2t ++ mktemp + local LAST_OUT=/tmp/tmp.flq8RQ1WpR ++ mktemp + local LAST_ERR=/tmp/tmp.bBviKg6fR3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-6b86564996-69q2t + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.flq8RQ1WpR + cat /tmp/tmp.bBviKg6fR3 + rm /tmp/tmp.flq8RQ1WpR /tmp/tmp.bBviKg6fR3 + return 0 2025/10/09 00:09:58 http: TLS handshake error from 10.11.40.4:36926: EOF 2025-10-09T00:09:36.114Z INFO setup Manager starting up {"gitCommit": "c5e2e681577ec5135cf7e4f3a99405aadfebe3aa", "gitBranch": "PR-2202-c5e2e681", "buildTime": "2025-10-08T22:13:19Z", "goVersion": "go1.24.8", "os": "linux", "arch": "amd64"} 2025-10-09T00:09:36.114Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.31.13-gke.1008000"} 2025-10-09T00:09:36.117Z INFO setup Registering Components. 2025-10-09T00:09:40.791Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2025-10-09T00:09:40.791Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2025-10-09T00:09:40.791Z INFO controller-runtime.metrics Starting metrics server 2025-10-09T00:09:40.791Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2025-10-09T00:09:40.791Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-10-09T00:09:40.791Z INFO controller-runtime.webhook Starting webhook server 2025-10-09T00:09:40.791Z INFO setup Starting the Cmd. 2025-10-09T00:09:40.791Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-10-09T00:09:40.792Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2025-10-09T00:09:40.892Z INFO attempting to acquire leader lease pxc-operator/08db1feb.percona.com... 2025-10-09T00:09:57.487Z DEBUG events percona-xtradb-cluster-operator-6b86564996-69q2t_e87fb6bc-0b65-4834-b8ab-5b56b69cf2a7 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"0b838650-608f-4077-8f26-81cb51211674","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1759968597482031009"}, "reason": "LeaderElection"} 2025-10-09T00:09:57.487Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.Secret"} 2025-10-09T00:09:57.487Z INFO successfully acquired lease pxc-operator/08db1feb.percona.com 2025-10-09T00:09:57.488Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2025-10-09T00:09:57.488Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"} 2025-10-09T00:09:57.488Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2025-10-09T00:09:57.588Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2025-10-09T00:09:57.588Z INFO Starting Controller {"controller": "pxc-controller"} 2025-10-09T00:09:57.588Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2025-10-09T00:09:57.588Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2025-10-09T00:09:57.588Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2025-10-09T00:09:57.588Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2025-10-09T00:09:58.679Z ERROR Reconciler error {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "minimal-cluster", "reconcileID": "abe03eb6-0538-4a2e-981a-7f2b67b55f66", "error": "failed to update cr finalizers: Internal error occurred: failed calling webhook \"validationwebhook.pxc.percona.com\": failed to call webhook: Post \"https://percona-xtradb-cluster-operator.pxc-operator.svc:443/validate-percona-xtradbcluster?timeout=10s\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \"crypto/rsa: verification error\" while trying to verify candidate authority certificate \"Root CA\")", "errorVerbose": "Internal error occurred: failed calling webhook \"validationwebhook.pxc.percona.com\": failed to call webhook: Post \"https://percona-xtradb-cluster-operator.pxc-operator.svc:443/validate-percona-xtradbcluster?timeout=10s\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \"crypto/rsa: verification error\" while trying to verify candidate authority certificate \"Root CA\")\nfailed to update cr finalizers\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:307\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} 2025-10-09T00:10:03.279Z ERROR Update status {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "minimal-cluster", "reconcileID": "dfc77bd9-3e6e-4d56-8064-63d6bb7b7f78", "error": "PerconaXtraDBCluster.pxc.percona.com \"minimal-cluster\" not found"} 2025-10-09T00:10:37.694Z INFO spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`. 2025-10-09T00:10:40.767Z INFO spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`. 2025-10-09T00:10:40.796Z INFO spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`. 2025-10-09T00:10:43.897Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "46833ae6-c74b-496e-8a0e-a63bfb5d5b11", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2025-10-09T00:10:43.918Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "46833ae6-c74b-496e-8a0e-a63bfb5d5b11", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2025-10-09T00:10:44.483Z ERROR Reconciler error {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "46833ae6-c74b-496e-8a0e-a63bfb5d5b11", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:92\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:25\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:52\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:590\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:27\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:52\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:590\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:54\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:590\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:578\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:590\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:591\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} 2025-10-09T00:10:44.665Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "238d0ab2-f706-44e8-b861-427f0da6fdef", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2025-10-09T00:10:44.789Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "238d0ab2-f706-44e8-b861-427f0da6fdef", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2025-10-09T00:10:44.883Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "238d0ab2-f706-44e8-b861-427f0da6fdef", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-09T00:10:44.950Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "238d0ab2-f706-44e8-b861-427f0da6fdef", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-09T00:10:45.027Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "238d0ab2-f706-44e8-b861-427f0da6fdef", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-09T00:10:45.130Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "238d0ab2-f706-44e8-b861-427f0da6fdef", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-09T00:10:46.352Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "10f733d6-5967-447f-9413-9c485f6c1437", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2025-10-09T00:10:46.377Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "10f733d6-5967-447f-9413-9c485f6c1437", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2025-10-09T00:12:02.724Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "8644a637-354d-47a2-960f-61e42c97aa4b", "user": "operator"} 2025-10-09T00:12:02.761Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "8644a637-354d-47a2-960f-61e42c97aa4b", "user": "monitor"} 2025-10-09T00:12:02.806Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "8644a637-354d-47a2-960f-61e42c97aa4b"} 2025-10-09T00:12:02.835Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "8644a637-354d-47a2-960f-61e42c97aa4b"} 2025-10-09T00:12:02.869Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "8644a637-354d-47a2-960f-61e42c97aa4b", "user": "xtrabackup"} 2025-10-09T00:12:02.907Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "8644a637-354d-47a2-960f-61e42c97aa4b"} 2025-10-09T00:12:02.935Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "8644a637-354d-47a2-960f-61e42c97aa4b", "user": "replication"} 2025-10-09T00:14:29.486Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "fffc98c4-66ae-463b-a895-37dbba370671", "user": "root"} 2025-10-09T00:14:30.633Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "fffc98c4-66ae-463b-a895-37dbba370671", "new version": "8.0.19-10"} 2025-10-09T00:16:22.167Z INFO add new job {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "17102c3a-999e-44f4-9a85-fedb17f2925d", "name": "ensure-version/smart-update2-10649/smart-update", "schedule": "* * * * *"} 2025-10-09T00:17:22.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "17102c3a-999e-44f4-9a85-fedb17f2925d", "endpoint": "http://version-service.smart-update2-10649.svc.cluster.local:11000"} 2025-10-09T00:17:22.047Z ERROR failed to ensure version {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "17102c3a-999e-44f4-9a85-fedb17f2925d", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=e8f289b9-a522-4c90-86b3-9a9d24955d67&databaseVersion=8.0.19-10&kubeVersion=v1.31.13-gke.1008000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=e8f289b9-a522-4c90-86b3-9a9d24955d67&databaseVersion=8.0.19-10&kubeVersion=v1.31.13-gke.1008000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:300\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:302\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} 2025-10-09T00:18:22.001Z INFO cluster is not ready {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "17102c3a-999e-44f4-9a85-fedb17f2925d"} 2025-10-09T00:18:42.809Z ERROR Update status {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "0c17b2b2-17cc-474a-9942-c8902dbf00ad", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2025-10-09T00:19:00.666Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "75fbbe79-90bf-47f0-8614-fbb8fc008a42", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2025-10-09T00:19:00.681Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "75fbbe79-90bf-47f0-8614-fbb8fc008a42", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2025-10-09T00:19:01.269Z ERROR Reconciler error {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "75fbbe79-90bf-47f0-8614-fbb8fc008a42", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:92\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:25\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:52\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:590\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:27\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:52\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:590\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:54\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:590\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:578\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:590\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:591\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} 2025-10-09T00:19:01.385Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "6c44c8cb-327a-4c4d-a551-423478930021", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2025-10-09T00:19:01.431Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "6c44c8cb-327a-4c4d-a551-423478930021", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2025-10-09T00:19:01.499Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "6c44c8cb-327a-4c4d-a551-423478930021", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-09T00:19:01.568Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "6c44c8cb-327a-4c4d-a551-423478930021", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-09T00:19:01.690Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "6c44c8cb-327a-4c4d-a551-423478930021", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-09T00:19:02.007Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "6c44c8cb-327a-4c4d-a551-423478930021", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-09T00:19:02.741Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "6247f797-d0a6-4ed2-a222-4fffa3fd5cc8", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2025-10-09T00:19:02.762Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "6247f797-d0a6-4ed2-a222-4fffa3fd5cc8", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2025-10-09T00:22:47.060Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "b2859a75-f7e1-41d2-8b71-a13f46f1c3bc", "new version": "8.0.19-10"} 2025-10-09T00:24:17.089Z INFO add new job {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "14092829-1c17-4748-b8f8-45788186adc9", "name": "ensure-version/smart-update2-10649/smart-update", "schedule": "* * * * *"} 2025-10-09T00:25:18.111Z DEBUG Use version service endpoint {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "14092829-1c17-4748-b8f8-45788186adc9", "endpoint": "http://version-service.smart-update2-10649.svc.cluster.local:11000"} 2025-10-09T00:25:18.152Z ERROR failed to ensure version {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "14092829-1c17-4748-b8f8-45788186adc9", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=15e397d7-f4b9-4d67-b322-b3490f505352&databaseVersion=8.0.19-10&kubeVersion=v1.31.13-gke.1008000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=15e397d7-f4b9-4d67-b322-b3490f505352&databaseVersion=8.0.19-10&kubeVersion=v1.31.13-gke.1008000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:300\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:302\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} 2025-10-09T00:26:17.001Z INFO cluster is not ready {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "14092829-1c17-4748-b8f8-45788186adc9"} 2025-10-09T00:26:33.923Z ERROR Update status {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "30e4622f-8040-473d-8383-623748343136", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} 2025-10-09T00:26:52.683Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "18216472-feb5-41ad-813b-7b7aac68e372", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2025-10-09T00:26:52.705Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "18216472-feb5-41ad-813b-7b7aac68e372", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2025-10-09T00:26:53.281Z ERROR Reconciler error {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "18216472-feb5-41ad-813b-7b7aac68e372", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:92\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:25\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:52\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:590\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:27\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:52\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:590\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:54\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:590\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:578\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:590\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:591\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} 2025-10-09T00:26:53.402Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "a946e368-58c9-4daa-9693-f9096f1faaea", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2025-10-09T00:26:53.443Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "a946e368-58c9-4daa-9693-f9096f1faaea", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2025-10-09T00:26:53.494Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "a946e368-58c9-4daa-9693-f9096f1faaea", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-09T00:26:53.556Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "a946e368-58c9-4daa-9693-f9096f1faaea", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-09T00:26:53.616Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "a946e368-58c9-4daa-9693-f9096f1faaea", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-09T00:26:53.698Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "a946e368-58c9-4daa-9693-f9096f1faaea", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-09T00:26:54.834Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "b6a72f2f-4561-42b9-b4b9-6a9728c278f7", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2025-10-09T00:26:54.863Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "b6a72f2f-4561-42b9-b4b9-6a9728c278f7", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2025-10-09T00:30:44.733Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "5e74eb67-5990-4ce9-b65d-feb59a353b12", "new version": "8.0.19-10"} 2025-10-09T00:32:27.799Z INFO add new job {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "5ab6773c-f4d6-4a8b-b46f-e00bb21e2b41", "name": "ensure-version/smart-update2-10649/smart-update", "schedule": "* * * * *"} 2025-10-09T00:33:27.001Z DEBUG Use version service endpoint {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "5ab6773c-f4d6-4a8b-b46f-e00bb21e2b41", "endpoint": "http://version-service.smart-update2-10649.svc.cluster.local:11000"} 2025-10-09T00:33:27.042Z ERROR failed to ensure version {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "5ab6773c-f4d6-4a8b-b46f-e00bb21e2b41", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.19-10.1?clusterWideEnabled=true&customResourceUid=eb518e96-afaf-41d5-a068-313987362386&databaseVersion=8.0.19-10&kubeVersion=v1.31.13-gke.1008000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.19-10.1?clusterWideEnabled=true&customResourceUid=eb518e96-afaf-41d5-a068-313987362386&databaseVersion=8.0.19-10&kubeVersion=v1.31.13-gke.1008000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:300\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:302\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} 2025-10-09T00:34:27.001Z INFO cluster is not ready {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "5ab6773c-f4d6-4a8b-b46f-e00bb21e2b41"} 2025-10-09T00:34:34.827Z ERROR Update status {"controller": "pxc-controller", "namespace": "smart-update2-10649", "name": "smart-update", "reconcileID": "83c2f329-97cc-4063-8bbc-1b7f0d916da7", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1 github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1 github.com/robfig/cron/v3.(*Cron).startJob.func1 github.com/robfig/cron/v3.FuncJob.Run /go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136 /go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:258 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:311 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:214 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.r0o0Y1BQpO ++ mktemp + local LAST_ERR=/tmp/tmp.ErvHSjow9t + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.r0o0Y1BQpO No resources found + cat /tmp/tmp.ErvHSjow9t + rm /tmp/tmp.r0o0Y1BQpO /tmp/tmp.ErvHSjow9t + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.3GpjK2CMN8 ++ mktemp + local LAST_ERR=/tmp/tmp.A6V7k2xl7Y + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3GpjK2CMN8 No resources found + cat /tmp/tmp.A6V7k2xl7Y + rm /tmp/tmp.3GpjK2CMN8 /tmp/tmp.A6V7k2xl7Y + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.BgQG35UZz8 ++ mktemp + local LAST_ERR=/tmp/tmp.uvi6hfERbl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BgQG35UZz8 No resources found + cat /tmp/tmp.uvi6hfERbl + rm /tmp/tmp.BgQG35UZz8 /tmp/tmp.uvi6hfERbl + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.ZfJwMNPSj4 ++ mktemp + local LAST_ERR=/tmp/tmp.ujZStAVRoz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZfJwMNPSj4 validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.ujZStAVRoz + rm /tmp/tmp.ZfJwMNPSj4 /tmp/tmp.ujZStAVRoz + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.18.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace smart-update2-10649 + rm -rf /tmp/tmp.7JfPSw7Gpf + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.7SN0vt011n ++ mktemp + desc 'test passed' + local LAST_OUT=/tmp/tmp.CJLTnHFX4t + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- ++ mktemp + local LAST_ERR=/tmp/tmp.4XFDGlVyys + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.u83MkYz1jH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace smart-update2-10649 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator