Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/logs/smart-update1-8-0.log WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 + API=pxc.percona.com/v9-9-9 + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + CLUSTER=smart-update + CLUSTER_SIZE=3 + PROXY_SIZE=2 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + PXC_VER=8.0 + TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + VS_URL=http://version-service + VS_PORT=11000 + VS_ENDPOINT=http://version-service:11000 + main + create_infra smart-update1-19776 + local ns=smart-update1-19776 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n smart-update1-30993 smart-update --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/smart-update patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.Df5fAMd5Gy ++ mktemp + local LAST_ERR=/tmp/tmp.pzyVOX4Bro + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Df5fAMd5Gy perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.pzyVOX4Bro + rm /tmp/tmp.Df5fAMd5Gy /tmp/tmp.pzyVOX4Bro + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.3m4i6xSw3Q ++ mktemp + local LAST_ERR=/tmp/tmp.hPyYjSA7Lm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3m4i6xSw3Q No resources found + cat /tmp/tmp.hPyYjSA7Lm + rm /tmp/tmp.3m4i6xSw3Q /tmp/tmp.hPyYjSA7Lm + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.AVSmZoIC4E ++ mktemp + local LAST_ERR=/tmp/tmp.MNibv0bG7n + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AVSmZoIC4E No resources found + cat /tmp/tmp.MNibv0bG7n + rm /tmp/tmp.AVSmZoIC4E /tmp/tmp.MNibv0bG7n + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ awk '-F ' '{print $2}' ++ tail -n1 ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + xargs kubectl delete ns ++ mktemp + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.mQxQPYp5Wb ++ mktemp + local LAST_ERR=/tmp/tmp.0PVmA0o9te + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + local LAST_OUT=/tmp/tmp.nDoqHIUzsf + set +e + kubectl get ns + awk '{print$1}' ++ mktemp + local LAST_ERR=/tmp/tmp.t6jfCBWN3e + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mQxQPYp5Wb + cat /tmp/tmp.0PVmA0o9te + rm /tmp/tmp.mQxQPYp5Wb /tmp/tmp.0PVmA0o9te + return 0 namespace "cert-manager" deleted namespace "smart-update1-30993" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nDoqHIUzsf namespace "pxc-operator" deleted + cat /tmp/tmp.t6jfCBWN3e + rm /tmp/tmp.nDoqHIUzsf /tmp/tmp.t6jfCBWN3e + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'namespace/pxc-operator - ' namespace/pxc-operator - + set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.tw3JCFKdFa ++ mktemp + local LAST_ERR=/tmp/tmp.s5t4rpLMhM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tw3JCFKdFa namespace/pxc-operator created + cat /tmp/tmp.s5t4rpLMhM + rm /tmp/tmp.tw3JCFKdFa /tmp/tmp.s5t4rpLMhM + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.XweJZYFvZO +++ mktemp ++ local LAST_ERR=/tmp/tmp.exarGmkAe9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XweJZYFvZO ++ cat /tmp/tmp.exarGmkAe9 ++ rm /tmp/tmp.XweJZYFvZO /tmp/tmp.exarGmkAe9 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1737-0ed7c642-6-cluster9 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.5qwYAwNqXq ++ mktemp + local LAST_ERR=/tmp/tmp.wBubCc3mi3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1737-0ed7c642-6-cluster9 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5qwYAwNqXq Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1737-0ed7c642-6-cluster9" modified. + cat /tmp/tmp.wBubCc3mi3 + rm /tmp/tmp.5qwYAwNqXq /tmp/tmp.wBubCc3mi3 + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.YKvdrKYgtJ ++ mktemp + local LAST_ERR=/tmp/tmp.FjyKXHcObQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YKvdrKYgtJ customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.FjyKXHcObQ + rm /tmp/tmp.YKvdrKYgtJ /tmp/tmp.FjyKXHcObQ + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/deploy/cw-rbac.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.rWxt1DSnUY ++ mktemp + local LAST_ERR=/tmp/tmp.4Hs5y78cOk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rWxt1DSnUY clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.4Hs5y78cOk + rm /tmp/tmp.rWxt1DSnUY /tmp/tmp.4Hs5y78cOk + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/deploy/cw-operator.yaml + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - ++ mktemp + local LAST_OUT=/tmp/tmp.RJsw6WoWdM ++ mktemp + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1737-0ed7c642^' + local LAST_ERR=/tmp/tmp.bVScpaVOVc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RJsw6WoWdM deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.bVScpaVOVc + rm /tmp/tmp.RJsw6WoWdM /tmp/tmp.bVScpaVOVc + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.jnRyOjWv4d ++ mktemp + local LAST_ERR=/tmp/tmp.0S7xpRLf73 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jnRyOjWv4d pod/percona-xtradb-cluster-operator-6c9cbdddbf-qls6l condition met + cat /tmp/tmp.0S7xpRLf73 + rm /tmp/tmp.jnRyOjWv4d /tmp/tmp.0S7xpRLf73 + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.MTNmpzID8s +++ mktemp ++ local LAST_ERR=/tmp/tmp.XVBvvVymIJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MTNmpzID8s ++ cat /tmp/tmp.XVBvvVymIJ ++ rm /tmp/tmp.MTNmpzID8s /tmp/tmp.XVBvvVymIJ ++ return 0 + wait_pod percona-xtradb-cluster-operator-6c9cbdddbf-qls6l 480 pxc-operator + local pod=percona-xtradb-cluster-operator-6c9cbdddbf-qls6l + local max_retry=480 + local ns=pxc-operator ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo percona-xtradb-cluster-operator-6c9cbdddbf-qls6l + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-6c9cbdddbf-qls6l condition met percona-xtradb-cluster-operator-6c9cbdddbf-qls6l.Ok + sleep 3 + create_namespace smart-update1-19776 + local namespace=smart-update1-19776 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ tail -n1 ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces smart-update1-19776' + set +o xtrace + xargs kubectl delete ns ----------------------------------------------------------------------------------- cleaned up old namespaces smart-update1-19776 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace smart-update1-19776 + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.JvAoLgVEzC ++ mktemp + local LAST_ERR=/tmp/tmp.hAeOBKjMtE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-19776 + local LAST_OUT=/tmp/tmp.7CUa1Nd4Vn + awk '{print$1}' ++ mktemp + local LAST_ERR=/tmp/tmp.emVRpmNm0L + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-19776 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7CUa1Nd4Vn + cat /tmp/tmp.emVRpmNm0L + rm /tmp/tmp.7CUa1Nd4Vn /tmp/tmp.emVRpmNm0L + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-19776 Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.JvAoLgVEzC + cat /tmp/tmp.hAeOBKjMtE Error from server (NotFound): namespaces "smart-update1-19776" not found + rm /tmp/tmp.JvAoLgVEzC /tmp/tmp.hAeOBKjMtE + return 1 + : + wait_for_delete namespace/smart-update1-19776 + local res=namespace/smart-update1-19776 + echo -n 'namespace/smart-update1-19776 - ' namespace/smart-update1-19776 - + set +o xtrace Error from server (NotFound): namespaces "smart-update1-19776" not found + desc 'create namespace smart-update1-19776' + set +o xtrace ----------------------------------------------------------------------------------- create namespace smart-update1-19776 ----------------------------------------------------------------------------------- + kubectl_bin create namespace smart-update1-19776 ++ mktemp + local LAST_OUT=/tmp/tmp.gTFVYx8eMt ++ mktemp + local LAST_ERR=/tmp/tmp.dHoqe0raXm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace smart-update1-19776 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gTFVYx8eMt namespace/smart-update1-19776 created + cat /tmp/tmp.dHoqe0raXm + rm /tmp/tmp.gTFVYx8eMt /tmp/tmp.dHoqe0raXm + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Tb2wntadej +++ mktemp ++ local LAST_ERR=/tmp/tmp.PkF7yHXeaU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Tb2wntadej ++ cat /tmp/tmp.PkF7yHXeaU ++ rm /tmp/tmp.Tb2wntadej /tmp/tmp.PkF7yHXeaU ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1737-0ed7c642-6-cluster9 --namespace=smart-update1-19776 ++ mktemp + local LAST_OUT=/tmp/tmp.qP6TEO4kLX ++ mktemp + local LAST_ERR=/tmp/tmp.6rjuMLFePN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1737-0ed7c642-6-cluster9 --namespace=smart-update1-19776 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qP6TEO4kLX Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1737-0ed7c642-6-cluster9" modified. + cat /tmp/tmp.6rjuMLFePN + rm /tmp/tmp.qP6TEO4kLX /tmp/tmp.6rjuMLFePN + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.vBHDrWRe1D ++ mktemp + local LAST_ERR=/tmp/tmp.kceXFV1dS1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vBHDrWRe1D secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.kceXFV1dS1 + rm /tmp/tmp.vBHDrWRe1D /tmp/tmp.kceXFV1dS1 + return 0 + deploy_version_service + desc 'install version service' + set +o xtrace ----------------------------------------------------------------------------------- install version service ----------------------------------------------------------------------------------- + kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json ++ mktemp + local LAST_OUT=/tmp/tmp.DkUKzRAXz5 ++ mktemp + local LAST_ERR=/tmp/tmp.blfJLrPmUX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DkUKzRAXz5 configmap/versions created + cat /tmp/tmp.blfJLrPmUX + rm /tmp/tmp.DkUKzRAXz5 /tmp/tmp.blfJLrPmUX + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.kv33qPtWbv ++ mktemp + local LAST_ERR=/tmp/tmp.IkTBqsEQao + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kv33qPtWbv deployment.apps/version-service created service/version-service created + cat /tmp/tmp.IkTBqsEQao + rm /tmp/tmp.kv33qPtWbv /tmp/tmp.IkTBqsEQao + return 0 + sleep 10 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.1C28lu4ojM ++ mktemp + local LAST_ERR=/tmp/tmp.P3jlgSMOyo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1C28lu4ojM namespace/cert-manager created + cat /tmp/tmp.P3jlgSMOyo + rm /tmp/tmp.1C28lu4ojM /tmp/tmp.P3jlgSMOyo + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.T0VHvW979u ++ mktemp + local LAST_ERR=/tmp/tmp.IlwzNkS6sj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.T0VHvW979u namespace/cert-manager labeled + cat /tmp/tmp.IlwzNkS6sj + rm /tmp/tmp.T0VHvW979u /tmp/tmp.IlwzNkS6sj + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.iM8TBASdA6 ++ mktemp + local LAST_ERR=/tmp/tmp.3eAntCYOyy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iM8TBASdA6 namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews configured role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection configured rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.3eAntCYOyy Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.iM8TBASdA6 /tmp/tmp.3eAntCYOyy + return 0 + '[' '' == 4.10 ']' + sleep 70 ++ head -n1 +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++++ grep -c percona-xtradb-cluster-operator ++ tail -n3 ++ grep :8.0 ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ sort -V +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zluaKNs2n1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FQHkqsFoa9 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.zluaKNs2n1 +++ cat /tmp/tmp.FQHkqsFoa9 +++ rm /tmp/tmp.zluaKNs2n1 /tmp/tmp.FQHkqsFoa9 +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-6c9cbdddbf-qls6l -n pxc-operator -- curl -s http://version-service.smart-update1-19776.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.A0quFxxHDM +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ro0teojY9w ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-6c9cbdddbf-qls6l -n pxc-operator -- curl -s http://version-service.smart-update1-19776.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A0quFxxHDM ++ cat /tmp/tmp.Ro0teojY9w Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.A0quFxxHDM /tmp/tmp.Ro0teojY9w ++ return 0 + IMAGE_PXC=percona/percona-xtradb-cluster:8.0.19-10.1 + desc 'patch crd' + set +o xtrace ----------------------------------------------------------------------------------- patch crd ----------------------------------------------------------------------------------- + kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' ++ mktemp + local LAST_OUT=/tmp/tmp.5KmclyjeBI ++ mktemp + local LAST_ERR=/tmp/tmp.8YAEhZ3NpD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5KmclyjeBI customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched + cat /tmp/tmp.8YAEhZ3NpD + rm /tmp/tmp.5KmclyjeBI /tmp/tmp.8YAEhZ3NpD + return 0 + desc 'Updating ProxySQL PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- Updating ProxySQL PXC cluster ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/conf/smart-update.yml /tmp/tmp.j6tvA4WuUZ/smart-update.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1737-0ed7c642"' /tmp/tmp.j6tvA4WuUZ/smart-update.yml + spinup_pxc smart-update /tmp/tmp.j6tvA4WuUZ/smart-update.yml + local cluster=smart-update + local config=/tmp/tmp.j6tvA4WuUZ/smart-update.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.j0B441Roex ++ mktemp + local LAST_ERR=/tmp/tmp.n9l8Bal1sc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.j0B441Roex secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.n9l8Bal1sc + rm /tmp/tmp.j0B441Roex /tmp/tmp.n9l8Bal1sc + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/client.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + local LAST_OUT=/tmp/tmp.0qScx5YQtK + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1737-0ed7c642#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-19776~ + local LAST_ERR=/tmp/tmp.GTeMILpSvZ + local exit_status=0 + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0qScx5YQtK deployment.apps/pxc-client created + cat /tmp/tmp.GTeMILpSvZ + rm /tmp/tmp.0qScx5YQtK /tmp/tmp.GTeMILpSvZ + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.j6tvA4WuUZ/smart-update.yml + '[' -z '' ']' + cat_config /tmp/tmp.j6tvA4WuUZ/smart-update.yml + kubectl_bin apply -f - + cat /tmp/tmp.j6tvA4WuUZ/smart-update.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1737-0ed7c642#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.ktinPYjcmD + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-19776~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_ERR=/tmp/tmp.D4UHamTYcU + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ktinPYjcmD perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.D4UHamTYcU + rm /tmp/tmp.ktinPYjcmD /tmp/tmp.D4UHamTYcU + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QDUlSbLuH4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WRKlwKIrtJ +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.QDUlSbLuH4 +++ cat /tmp/tmp.WRKlwKIrtJ +++ rm /tmp/tmp.QDUlSbLuH4 /tmp/tmp.WRKlwKIrtJ +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yQNgctWBu9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1BGsHDYyRu +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.yQNgctWBu9 +++ cat /tmp/tmp.1BGsHDYyRu +++ rm /tmp/tmp.yQNgctWBu9 /tmp/tmp.1BGsHDYyRu +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-proxysql ++ return + local proxy=smart-update-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-19776 ++ mktemp + local LAST_OUT=/tmp/tmp.mMLLQYnNWT ++ mktemp + local LAST_ERR=/tmp/tmp.rCGxohspgl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-19776 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-19776 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-19776 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.mMLLQYnNWT + cat /tmp/tmp.rCGxohspgl error: no matching resources found + rm /tmp/tmp.mMLLQYnNWT /tmp/tmp.rCGxohspgl + return 1 + true + wait_for_running smart-update-proxysql 1 + local name=smart-update-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-proxysql-0 480 + local pod=smart-update-proxysql-0 + local max_retry=480 + local ns= ++ echo smart-update-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace Error from server (NotFound): pods "smart-update-proxysql-0" not found smart-update-proxysql-0.........Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met smart-update-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met smart-update-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-proxysql -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fqL02SOQ6N +++ mktemp ++ local LAST_ERR=/tmp/tmp.rZqBHudJD4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fqL02SOQ6N ++ cat /tmp/tmp.rZqBHudJD4 ++ rm /tmp/tmp.fqL02SOQ6N /tmp/tmp.rZqBHudJD4 ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-proxysql -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.msXP6aeYk1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.AjXwo2ymq2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.msXP6aeYk1 ++ cat /tmp/tmp.AjXwo2ymq2 ++ rm /tmp/tmp.msXP6aeYk1 /tmp/tmp.AjXwo2ymq2 ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ue1Wnw56x2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RirEuYbOlm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ue1Wnw56x2 ++ cat /tmp/tmp.RirEuYbOlm ++ rm /tmp/tmp.ue1Wnw56x2 /tmp/tmp.RirEuYbOlm ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2sXoE6VSRI +++ mktemp ++ local LAST_ERR=/tmp/tmp.KwbQmscmae ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2sXoE6VSRI ++ cat /tmp/tmp.KwbQmscmae ++ rm /tmp/tmp.2sXoE6VSRI /tmp/tmp.KwbQmscmae ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PlrR0EyGCU +++ mktemp ++ local LAST_ERR=/tmp/tmp.GYaD9paDAF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PlrR0EyGCU ++ cat /tmp/tmp.GYaD9paDAF ++ rm /tmp/tmp.PlrR0EyGCU /tmp/tmp.GYaD9paDAF ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NR0kz3MP75 +++ mktemp ++ local LAST_ERR=/tmp/tmp.OJEDIoKaqb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NR0kz3MP75 ++ cat /tmp/tmp.OJEDIoKaqb Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.NR0kz3MP75 /tmp/tmp.OJEDIoKaqb ++ return 0 + '[' '' ']' +++ get_proxy smart-update +++ local target_cluster=smart-update ++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.9NfQXHECAJ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.urDjBbB5zg ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.9NfQXHECAJ ++++ cat /tmp/tmp.urDjBbB5zg ++++ rm /tmp/tmp.9NfQXHECAJ /tmp/tmp.urDjBbB5zg ++++ return 0 +++ [[ '' == \t\r\u\e ]] ++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.96RI5UJusa +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.cm1pc84KXL ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.96RI5UJusa ++++ cat /tmp/tmp.cm1pc84KXL ++++ rm /tmp/tmp.96RI5UJusa /tmp/tmp.cm1pc84KXL ++++ return 0 +++ [[ true == \t\r\u\e ]] +++ echo smart-update-proxysql +++ return ++ get_proxy_primary '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=smart-update-proxysql-0 +++ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 proxysql +++ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' +++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' +++ local pod=smart-update-proxysql-0 +++ local container_name=proxysql +++ set +o xtrace ++ local ip=smart-update-pxc-0.smart-update-pxc.smart-update1-19776.svc.cluster.local +++ echo smart-update-pxc-0.smart-update-pxc.smart-update1-19776.svc.cluster.local +++ wc -l ++ '[' 1 '!=' 1 ']' ++ cut -d. -f1 ++ echo smart-update-pxc-0.smart-update-pxc.smart-update1-19776.svc.cluster.local + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.Tsh2ZVkALL ++ mktemp + local LAST_ERR=/tmp/tmp.yS3tp0fnPR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Tsh2ZVkALL perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.yS3tp0fnPR + rm /tmp/tmp.Tsh2ZVkALL /tmp/tmp.yS3tp0fnPR + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update.................................................................... smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lBuarahdWP +++ mktemp ++ local LAST_ERR=/tmp/tmp.g5dEcXOHiW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lBuarahdWP ++ cat /tmp/tmp.g5dEcXOHiW ++ rm /tmp/tmp.lBuarahdWP /tmp/tmp.g5dEcXOHiW ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kdIeKNuwtN +++ mktemp ++ local LAST_ERR=/tmp/tmp.hOtH5xNgjo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kdIeKNuwtN ++ cat /tmp/tmp.hOtH5xNgjo ++ rm /tmp/tmp.kdIeKNuwtN /tmp/tmp.hOtH5xNgjo ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UgIcyv8GkI +++ mktemp ++ local LAST_ERR=/tmp/tmp.kUcIjmSDsn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UgIcyv8GkI ++ cat /tmp/tmp.kUcIjmSDsn ++ rm /tmp/tmp.UgIcyv8GkI /tmp/tmp.kUcIjmSDsn ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3On4NqLrha +++ mktemp ++ local LAST_ERR=/tmp/tmp.OLlJoYkMNC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3On4NqLrha ++ cat /tmp/tmp.OLlJoYkMNC ++ rm /tmp/tmp.3On4NqLrha /tmp/tmp.OLlJoYkMNC ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KtclSOLa7g +++ mktemp ++ local LAST_ERR=/tmp/tmp.VgoSm4yKav ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KtclSOLa7g ++ cat /tmp/tmp.VgoSm4yKav ++ rm /tmp/tmp.KtclSOLa7g /tmp/tmp.VgoSm4yKav ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lM7DrruyQN +++ mktemp ++ local LAST_ERR=/tmp/tmp.w9WC5CHUfz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lM7DrruyQN ++ cat /tmp/tmp.w9WC5CHUfz ++ rm /tmp/tmp.lM7DrruyQN /tmp/tmp.w9WC5CHUfz ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.drszVwsSwH +++ mktemp ++ local LAST_ERR=/tmp/tmp.iLktcD3pQ2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.drszVwsSwH ++ cat /tmp/tmp.iLktcD3pQ2 ++ rm /tmp/tmp.drszVwsSwH /tmp/tmp.iLktcD3pQ2 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FpJe63Eb1U +++ mktemp ++ local LAST_ERR=/tmp/tmp.e1ppS3nJ60 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FpJe63Eb1U ++ cat /tmp/tmp.e1ppS3nJ60 ++ rm /tmp/tmp.FpJe63Eb1U /tmp/tmp.e1ppS3nJ60 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AhXkIjwnVg +++ mktemp ++ local LAST_ERR=/tmp/tmp.AS8yJ8qER3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AhXkIjwnVg ++ cat /tmp/tmp.AS8yJ8qER3 ++ rm /tmp/tmp.AhXkIjwnVg /tmp/tmp.AS8yJ8qER3 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 8 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vJMevBgBCU +++ mktemp ++ local LAST_ERR=/tmp/tmp.TkK7OntQpM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vJMevBgBCU ++ cat /tmp/tmp.TkK7OntQpM ++ rm /tmp/tmp.vJMevBgBCU /tmp/tmp.TkK7OntQpM ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 9 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RcYPbGoeSM +++ mktemp ++ local LAST_ERR=/tmp/tmp.GMmodOKVtl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RcYPbGoeSM ++ cat /tmp/tmp.GMmodOKVtl ++ rm /tmp/tmp.RcYPbGoeSM /tmp/tmp.GMmodOKVtl ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 10 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3tMzchOqZo +++ mktemp ++ local LAST_ERR=/tmp/tmp.Uj3E5yBa7L ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3tMzchOqZo ++ cat /tmp/tmp.Uj3E5yBa7L ++ rm /tmp/tmp.3tMzchOqZo /tmp/tmp.Uj3E5yBa7L ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 11 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xAj80lMqg8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.H5G0Dehcqy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xAj80lMqg8 ++ cat /tmp/tmp.H5G0Dehcqy ++ rm /tmp/tmp.xAj80lMqg8 /tmp/tmp.H5G0Dehcqy ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 12 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zSgfIc6oHo +++ mktemp ++ local LAST_ERR=/tmp/tmp.unAS0Mir0S ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zSgfIc6oHo ++ cat /tmp/tmp.unAS0Mir0S ++ rm /tmp/tmp.zSgfIc6oHo /tmp/tmp.unAS0Mir0S ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 13 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UWmsbgWdjt +++ mktemp ++ local LAST_ERR=/tmp/tmp.dUN1plhnv1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UWmsbgWdjt ++ cat /tmp/tmp.dUN1plhnv1 ++ rm /tmp/tmp.UWmsbgWdjt /tmp/tmp.dUN1plhnv1 ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.m6lXInhfaU +++ mktemp ++ local LAST_ERR=/tmp/tmp.gb5kQUc3mE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.m6lXInhfaU ++ cat /tmp/tmp.gb5kQUc3mE ++ rm /tmp/tmp.m6lXInhfaU /tmp/tmp.gb5kQUc3mE ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.V2g75ZtY6X ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.il9nfjuU1N +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.V2g75ZtY6X +++++ cat /tmp/tmp.il9nfjuU1N +++++ rm /tmp/tmp.V2g75ZtY6X /tmp/tmp.il9nfjuU1N +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.nm4BRIfee3 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.RGjr3iQGyw +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.nm4BRIfee3 +++++ cat /tmp/tmp.RGjr3iQGyw +++++ rm /tmp/tmp.nm4BRIfee3 /tmp/tmp.RGjr3iQGyw +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-proxysql ++++ return +++ local cluster_proxy=smart-update-proxysql +++ echo proxysql ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.F4B8d15zt8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.eMNgGVr2Hz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.F4B8d15zt8 ++ cat /tmp/tmp.eMNgGVr2Hz ++ rm /tmp/tmp.F4B8d15zt8 /tmp/tmp.eMNgGVr2Hz ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SIQ8qzJSUu +++ mktemp ++ local LAST_ERR=/tmp/tmp.HrMAnJrGav ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SIQ8qzJSUu ++ cat /tmp/tmp.HrMAnJrGav ++ rm /tmp/tmp.SIQ8qzJSUu /tmp/tmp.HrMAnJrGav ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qRUjNCUIPm +++ mktemp ++ local LAST_ERR=/tmp/tmp.fFvgEA2GIL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qRUjNCUIPm ++ cat /tmp/tmp.fFvgEA2GIL ++ rm /tmp/tmp.qRUjNCUIPm /tmp/tmp.fFvgEA2GIL ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GlA7eUZ2vr +++ mktemp ++ local LAST_ERR=/tmp/tmp.QYdt8z3Tvw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GlA7eUZ2vr ++ cat /tmp/tmp.QYdt8z3Tvw ++ rm /tmp/tmp.GlA7eUZ2vr /tmp/tmp.QYdt8z3Tvw ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/conf/smart-update.yml ++ mktemp + local LAST_OUT=/tmp/tmp.zUxXebDqF8 ++ mktemp + local LAST_ERR=/tmp/tmp.nAiccdp4jT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/conf/smart-update.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zUxXebDqF8 perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.nAiccdp4jT + rm /tmp/tmp.zUxXebDqF8 /tmp/tmp.nAiccdp4jT + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.btNaWiwKC8 ++ mktemp + local LAST_ERR=/tmp/tmp.L1bP0kDukd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.btNaWiwKC8 persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted persistentvolumeclaim "proxydata-smart-update-proxysql-0" deleted persistentvolumeclaim "proxydata-smart-update-proxysql-1" deleted + cat /tmp/tmp.L1bP0kDukd + rm /tmp/tmp.btNaWiwKC8 /tmp/tmp.L1bP0kDukd + return 0 + desc 'Updating HAProxy PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- Updating HAProxy PXC cluster ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/conf/smart-update-haproxy.yml /tmp/tmp.j6tvA4WuUZ/smart-update-haproxy.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1737-0ed7c642"' /tmp/tmp.j6tvA4WuUZ/smart-update-haproxy.yml + spinup_pxc smart-update /tmp/tmp.j6tvA4WuUZ/smart-update-haproxy.yml + local cluster=smart-update + local config=/tmp/tmp.j6tvA4WuUZ/smart-update-haproxy.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Yis1EvBsJD ++ mktemp + local LAST_ERR=/tmp/tmp.74EFei8v4u + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Yis1EvBsJD secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.74EFei8v4u + rm /tmp/tmp.Yis1EvBsJD /tmp/tmp.74EFei8v4u + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1737-0ed7c642#' + local LAST_OUT=/tmp/tmp.AelIXkUGnI ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_ERR=/tmp/tmp.BpAnW0q9oJ + local exit_status=0 + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-19776~ + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AelIXkUGnI deployment.apps/pxc-client unchanged + cat /tmp/tmp.BpAnW0q9oJ + rm /tmp/tmp.AelIXkUGnI /tmp/tmp.BpAnW0q9oJ + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.j6tvA4WuUZ/smart-update-haproxy.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.j6tvA4WuUZ/smart-update-haproxy.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + cat /tmp/tmp.j6tvA4WuUZ/smart-update-haproxy.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_OUT=/tmp/tmp.oVdYaBArPh + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1737-0ed7c642#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-19776~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.iH1VitRO0P + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oVdYaBArPh perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.iH1VitRO0P + rm /tmp/tmp.oVdYaBArPh /tmp/tmp.iH1VitRO0P + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5JJ4gmrWmw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Zhy0abTcwL +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.5JJ4gmrWmw +++ cat /tmp/tmp.Zhy0abTcwL +++ rm /tmp/tmp.5JJ4gmrWmw /tmp/tmp.Zhy0abTcwL +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-19776 ++ mktemp + local LAST_OUT=/tmp/tmp.aceZxwvSf9 ++ mktemp + local LAST_ERR=/tmp/tmp.M33dspaAdv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-19776 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-19776 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-19776 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.aceZxwvSf9 + cat /tmp/tmp.M33dspaAdv error: no matching resources found + rm /tmp/tmp.aceZxwvSf9 /tmp/tmp.M33dspaAdv + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met smart-update-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met smart-update-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met smart-update-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EqvQAyHvWT +++ mktemp ++ local LAST_ERR=/tmp/tmp.bJOQ3rIrLP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EqvQAyHvWT ++ cat /tmp/tmp.bJOQ3rIrLP ++ rm /tmp/tmp.EqvQAyHvWT /tmp/tmp.bJOQ3rIrLP ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BKleEqKl0V +++ mktemp ++ local LAST_ERR=/tmp/tmp.0bt7kCcdI5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BKleEqKl0V ++ cat /tmp/tmp.0bt7kCcdI5 ++ rm /tmp/tmp.BKleEqKl0V /tmp/tmp.0bt7kCcdI5 ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.N2U0GunX0V +++ mktemp ++ local LAST_ERR=/tmp/tmp.bxKoJ9yfaL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.N2U0GunX0V ++ cat /tmp/tmp.bxKoJ9yfaL ++ rm /tmp/tmp.N2U0GunX0V /tmp/tmp.bxKoJ9yfaL ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.arAHfPa4gV +++ mktemp ++ local LAST_ERR=/tmp/tmp.OIUFkiOKE9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.arAHfPa4gV ++ cat /tmp/tmp.OIUFkiOKE9 ++ rm /tmp/tmp.arAHfPa4gV /tmp/tmp.OIUFkiOKE9 ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tuIGAwfmfF +++ mktemp ++ local LAST_ERR=/tmp/tmp.iY78TsxvaD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tuIGAwfmfF ++ cat /tmp/tmp.iY78TsxvaD ++ rm /tmp/tmp.tuIGAwfmfF /tmp/tmp.iY78TsxvaD ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mAwo1FKDbo +++ mktemp ++ local LAST_ERR=/tmp/tmp.gauicnylsw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mAwo1FKDbo ++ cat /tmp/tmp.gauicnylsw Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.mAwo1FKDbo /tmp/tmp.gauicnylsw ++ return 0 + '[' '' ']' ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kTZwNtlk7Z ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YenReZwC42 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.kTZwNtlk7Z +++ cat /tmp/tmp.YenReZwC42 +++ rm /tmp/tmp.kTZwNtlk7Z /tmp/tmp.YenReZwC42 +++ return 0 ++ client_pod=pxc-client-6644d8898f-h5hdr ++ wait_pod pxc-client-6644d8898f-h5hdr ++ local pod=pxc-client-6644d8898f-h5hdr ++ local max_retry=480 ++ local ns= +++ echo pxc-client-6644d8898f-h5hdr +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.wQv3IlzJ8N ++ mktemp + local LAST_ERR=/tmp/tmp.fiLhI3lsVp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wQv3IlzJ8N perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.fiLhI3lsVp + rm /tmp/tmp.wQv3IlzJ8N /tmp/tmp.fiLhI3lsVp + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update.................................................. smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6duxjsFP3M +++ mktemp ++ local LAST_ERR=/tmp/tmp.7wU1ZPgMoB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6duxjsFP3M ++ cat /tmp/tmp.7wU1ZPgMoB ++ rm /tmp/tmp.6duxjsFP3M /tmp/tmp.7wU1ZPgMoB ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9PKrrME0kM +++ mktemp ++ local LAST_ERR=/tmp/tmp.4hkLtSYfcL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9PKrrME0kM ++ cat /tmp/tmp.4hkLtSYfcL ++ rm /tmp/tmp.9PKrrME0kM /tmp/tmp.4hkLtSYfcL ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2FI3TscY0x +++ mktemp ++ local LAST_ERR=/tmp/tmp.oZbA3MeePb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2FI3TscY0x ++ cat /tmp/tmp.oZbA3MeePb ++ rm /tmp/tmp.2FI3TscY0x /tmp/tmp.oZbA3MeePb ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eqtw8yJjLk +++ mktemp ++ local LAST_ERR=/tmp/tmp.iP3QBSTvBd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eqtw8yJjLk ++ cat /tmp/tmp.iP3QBSTvBd ++ rm /tmp/tmp.eqtw8yJjLk /tmp/tmp.iP3QBSTvBd ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.o7o39Cp621 +++ mktemp ++ local LAST_ERR=/tmp/tmp.NuaV7H7C1E ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.o7o39Cp621 ++ cat /tmp/tmp.NuaV7H7C1E ++ rm /tmp/tmp.o7o39Cp621 /tmp/tmp.NuaV7H7C1E ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YZk6VZssJz +++ mktemp ++ local LAST_ERR=/tmp/tmp.G1L5R52SUR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YZk6VZssJz ++ cat /tmp/tmp.G1L5R52SUR ++ rm /tmp/tmp.YZk6VZssJz /tmp/tmp.G1L5R52SUR ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hDV15InJxs +++ mktemp ++ local LAST_ERR=/tmp/tmp.K49dgfQnen ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hDV15InJxs ++ cat /tmp/tmp.K49dgfQnen ++ rm /tmp/tmp.hDV15InJxs /tmp/tmp.K49dgfQnen ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U7i6cwOfK0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JHmBW6TkMt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.U7i6cwOfK0 ++ cat /tmp/tmp.JHmBW6TkMt ++ rm /tmp/tmp.U7i6cwOfK0 /tmp/tmp.JHmBW6TkMt ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.StmbF0VHAB +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZqQPP5uM8l ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.StmbF0VHAB ++ cat /tmp/tmp.ZqQPP5uM8l ++ rm /tmp/tmp.StmbF0VHAB /tmp/tmp.ZqQPP5uM8l ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.aTW3lPBbCf ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.f2Ha6FQs37 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.aTW3lPBbCf +++++ cat /tmp/tmp.f2Ha6FQs37 +++++ rm /tmp/tmp.aTW3lPBbCf /tmp/tmp.f2Ha6FQs37 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0b3RVmfqhQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.LffCia4oAf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0b3RVmfqhQ ++ cat /tmp/tmp.LffCia4oAf ++ rm /tmp/tmp.0b3RVmfqhQ /tmp/tmp.LffCia4oAf ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UX1mVLZzb2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.pBPQWFuz8K ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UX1mVLZzb2 ++ cat /tmp/tmp.pBPQWFuz8K ++ rm /tmp/tmp.UX1mVLZzb2 /tmp/tmp.pBPQWFuz8K ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7VqpxY5x6d +++ mktemp ++ local LAST_ERR=/tmp/tmp.kQwz4l8gU8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7VqpxY5x6d ++ cat /tmp/tmp.kQwz4l8gU8 ++ rm /tmp/tmp.7VqpxY5x6d /tmp/tmp.kQwz4l8gU8 ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xsJOhmSHWJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.Y8zBpAMW6z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xsJOhmSHWJ ++ cat /tmp/tmp.Y8zBpAMW6z ++ rm /tmp/tmp.xsJOhmSHWJ /tmp/tmp.Y8zBpAMW6z ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + kubectl_bin delete -f /tmp/tmp.j6tvA4WuUZ/smart-update-haproxy.yml ++ mktemp + local LAST_OUT=/tmp/tmp.yKQVggbiU9 ++ mktemp + local LAST_ERR=/tmp/tmp.xCXECdTETf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.j6tvA4WuUZ/smart-update-haproxy.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yKQVggbiU9 perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.xCXECdTETf + rm /tmp/tmp.yKQVggbiU9 /tmp/tmp.xCXECdTETf + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.MsHAS8l9cf ++ mktemp + local LAST_ERR=/tmp/tmp.5NvOx5Ilka + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MsHAS8l9cf persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.5NvOx5Ilka + rm /tmp/tmp.MsHAS8l9cf /tmp/tmp.5NvOx5Ilka + return 0 + desc 'Updating PXC cluster with version service available but disabled' + set +o xtrace ----------------------------------------------------------------------------------- Updating PXC cluster with version service available but disabled ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/conf/smart-update-version-service-reachable.yml /tmp/tmp.j6tvA4WuUZ/smart-update-version-service-reachable.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1737-0ed7c642"' /tmp/tmp.j6tvA4WuUZ/smart-update-version-service-reachable.yml + spinup_pxc smart-update /tmp/tmp.j6tvA4WuUZ/smart-update-version-service-reachable.yml + local cluster=smart-update + local config=/tmp/tmp.j6tvA4WuUZ/smart-update-version-service-reachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.zQbAlvAaaW ++ mktemp + local LAST_ERR=/tmp/tmp.wujqkVdg75 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zQbAlvAaaW secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.wujqkVdg75 + rm /tmp/tmp.zQbAlvAaaW /tmp/tmp.wujqkVdg75 + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1737-0ed7c642#' + local LAST_OUT=/tmp/tmp.N1JIfIv6gp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_ERR=/tmp/tmp.FdcJwrMNy2 + local exit_status=0 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-19776~ ++ seq 0 2 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.N1JIfIv6gp deployment.apps/pxc-client unchanged + cat /tmp/tmp.FdcJwrMNy2 + rm /tmp/tmp.N1JIfIv6gp /tmp/tmp.FdcJwrMNy2 + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.j6tvA4WuUZ/smart-update-version-service-reachable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.j6tvA4WuUZ/smart-update-version-service-reachable.yml + cat /tmp/tmp.j6tvA4WuUZ/smart-update-version-service-reachable.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.pMEnKjbWXU + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-19776~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1737-0ed7c642#' + local LAST_ERR=/tmp/tmp.pCzQJ0ezLN + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pMEnKjbWXU perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.pCzQJ0ezLN + rm /tmp/tmp.pMEnKjbWXU /tmp/tmp.pCzQJ0ezLN + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kPysc4jkYO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.dxoVtOFTaZ +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.kPysc4jkYO +++ cat /tmp/tmp.dxoVtOFTaZ +++ rm /tmp/tmp.kPysc4jkYO /tmp/tmp.dxoVtOFTaZ +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-19776 ++ mktemp + local LAST_OUT=/tmp/tmp.NIwiDTEKHd ++ mktemp + local LAST_ERR=/tmp/tmp.eFqO56WY5m + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-19776 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-19776 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-19776 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.NIwiDTEKHd + cat /tmp/tmp.eFqO56WY5m error: no matching resources found + rm /tmp/tmp.NIwiDTEKHd /tmp/tmp.eFqO56WY5m + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met smart-update-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met smart-update-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met smart-update-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KcjoTKud9v +++ mktemp ++ local LAST_ERR=/tmp/tmp.16RLxOh4An ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KcjoTKud9v ++ cat /tmp/tmp.16RLxOh4An ++ rm /tmp/tmp.KcjoTKud9v /tmp/tmp.16RLxOh4An ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uxn1GpDRow +++ mktemp ++ local LAST_ERR=/tmp/tmp.BVniVTWA9A ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uxn1GpDRow ++ cat /tmp/tmp.BVniVTWA9A ++ rm /tmp/tmp.uxn1GpDRow /tmp/tmp.BVniVTWA9A ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ml6yvrGyi0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ECKplYKGFj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ml6yvrGyi0 ++ cat /tmp/tmp.ECKplYKGFj ++ rm /tmp/tmp.ml6yvrGyi0 /tmp/tmp.ECKplYKGFj ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pbBm8U2VGn +++ mktemp ++ local LAST_ERR=/tmp/tmp.izvki3vL30 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pbBm8U2VGn ++ cat /tmp/tmp.izvki3vL30 ++ rm /tmp/tmp.pbBm8U2VGn /tmp/tmp.izvki3vL30 ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1j8nzCegWj +++ mktemp ++ local LAST_ERR=/tmp/tmp.Jx9OMl6pbf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1j8nzCegWj ++ cat /tmp/tmp.Jx9OMl6pbf ++ rm /tmp/tmp.1j8nzCegWj /tmp/tmp.Jx9OMl6pbf ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z5ztFtDszF +++ mktemp ++ local LAST_ERR=/tmp/tmp.YPYPAWs7rp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Z5ztFtDszF ++ cat /tmp/tmp.YPYPAWs7rp Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.Z5ztFtDszF /tmp/tmp.YPYPAWs7rp ++ return 0 + '[' '' ']' ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ovSDb8cOX4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fiGUjwG2h8 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.ovSDb8cOX4 +++ cat /tmp/tmp.fiGUjwG2h8 +++ rm /tmp/tmp.ovSDb8cOX4 /tmp/tmp.fiGUjwG2h8 +++ return 0 ++ client_pod=pxc-client-6644d8898f-h5hdr ++ wait_pod pxc-client-6644d8898f-h5hdr ++ local pod=pxc-client-6644d8898f-h5hdr ++ local max_retry=480 ++ local ns= +++ echo pxc-client-6644d8898f-h5hdr +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.gWcdCxj5Gg ++ mktemp + local LAST_ERR=/tmp/tmp.0JhAShglt9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gWcdCxj5Gg perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.0JhAShglt9 + rm /tmp/tmp.gWcdCxj5Gg /tmp/tmp.0JhAShglt9 + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update............................................ smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.O6vhJRfAwt +++ mktemp ++ local LAST_ERR=/tmp/tmp.IEvYOQpOjG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.O6vhJRfAwt ++ cat /tmp/tmp.IEvYOQpOjG ++ rm /tmp/tmp.O6vhJRfAwt /tmp/tmp.IEvYOQpOjG ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.24IKVF4vVa +++ mktemp ++ local LAST_ERR=/tmp/tmp.okvcpLA6sH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.24IKVF4vVa ++ cat /tmp/tmp.okvcpLA6sH ++ rm /tmp/tmp.24IKVF4vVa /tmp/tmp.okvcpLA6sH ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zalA35ByXp +++ mktemp ++ local LAST_ERR=/tmp/tmp.nDG57rthPC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zalA35ByXp ++ cat /tmp/tmp.nDG57rthPC ++ rm /tmp/tmp.zalA35ByXp /tmp/tmp.nDG57rthPC ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Sqwj9uVVzz +++ mktemp ++ local LAST_ERR=/tmp/tmp.0hMGxUjajj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Sqwj9uVVzz ++ cat /tmp/tmp.0hMGxUjajj ++ rm /tmp/tmp.Sqwj9uVVzz /tmp/tmp.0hMGxUjajj ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EzL1z8VSEv +++ mktemp ++ local LAST_ERR=/tmp/tmp.BeIlAVkxcZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EzL1z8VSEv ++ cat /tmp/tmp.BeIlAVkxcZ ++ rm /tmp/tmp.EzL1z8VSEv /tmp/tmp.BeIlAVkxcZ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mBTtPyXdBQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.9DqFhAfWED ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mBTtPyXdBQ ++ cat /tmp/tmp.9DqFhAfWED ++ rm /tmp/tmp.mBTtPyXdBQ /tmp/tmp.9DqFhAfWED ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0a8nMm2yXK +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ky9mZP30dT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0a8nMm2yXK ++ cat /tmp/tmp.Ky9mZP30dT ++ rm /tmp/tmp.0a8nMm2yXK /tmp/tmp.Ky9mZP30dT ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.S6Rd1JTlfy +++ mktemp ++ local LAST_ERR=/tmp/tmp.rVJr9tgJzh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.S6Rd1JTlfy ++ cat /tmp/tmp.rVJr9tgJzh ++ rm /tmp/tmp.S6Rd1JTlfy /tmp/tmp.rVJr9tgJzh ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.02Wuyv2qKz +++ mktemp ++ local LAST_ERR=/tmp/tmp.3aetAxDPcH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.02Wuyv2qKz ++ cat /tmp/tmp.3aetAxDPcH ++ rm /tmp/tmp.02Wuyv2qKz /tmp/tmp.3aetAxDPcH ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.y1WFOsIGvV +++ mktemp ++ local LAST_ERR=/tmp/tmp.RW4PcyRo2h ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.y1WFOsIGvV ++ cat /tmp/tmp.RW4PcyRo2h ++ rm /tmp/tmp.y1WFOsIGvV /tmp/tmp.RW4PcyRo2h ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.SUQGN9qDVj ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.aCMuouCZ10 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.SUQGN9qDVj +++++ cat /tmp/tmp.aCMuouCZ10 +++++ rm /tmp/tmp.SUQGN9qDVj /tmp/tmp.aCMuouCZ10 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cf7iRTmQpR +++ mktemp ++ local LAST_ERR=/tmp/tmp.klODIHeP1m ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cf7iRTmQpR ++ cat /tmp/tmp.klODIHeP1m ++ rm /tmp/tmp.cf7iRTmQpR /tmp/tmp.klODIHeP1m ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V7HB7A2G51 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4eKYTl3t1J ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.V7HB7A2G51 ++ cat /tmp/tmp.4eKYTl3t1J ++ rm /tmp/tmp.V7HB7A2G51 /tmp/tmp.4eKYTl3t1J ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aifRvSAJii +++ mktemp ++ local LAST_ERR=/tmp/tmp.W3r6UVmnzK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aifRvSAJii ++ cat /tmp/tmp.W3r6UVmnzK ++ rm /tmp/tmp.aifRvSAJii /tmp/tmp.W3r6UVmnzK ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pl0FX1d2bF +++ mktemp ++ local LAST_ERR=/tmp/tmp.0LWMvzdcJg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pl0FX1d2bF ++ cat /tmp/tmp.0LWMvzdcJg ++ rm /tmp/tmp.pl0FX1d2bF /tmp/tmp.0LWMvzdcJg ++ return 0 + client_pod=pxc-client-6644d8898f-h5hdr + wait_pod pxc-client-6644d8898f-h5hdr + local pod=pxc-client-6644d8898f-h5hdr + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-h5hdr ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-h5hdr condition met pxc-client-6644d8898f-h5hdr.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.j6tvA4WuUZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.j6tvA4WuUZ/select-1.sql + kubectl_bin delete -f /tmp/tmp.j6tvA4WuUZ/smart-update-version-service-reachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.VEtzLVwkwG ++ mktemp + local LAST_ERR=/tmp/tmp.5yBHXlRAUP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.j6tvA4WuUZ/smart-update-version-service-reachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VEtzLVwkwG perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.5yBHXlRAUP + rm /tmp/tmp.VEtzLVwkwG /tmp/tmp.5yBHXlRAUP + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.SWQt9ycAml ++ mktemp + local LAST_ERR=/tmp/tmp.ZETp9Cscxi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SWQt9ycAml persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.ZETp9Cscxi + rm /tmp/tmp.SWQt9ycAml /tmp/tmp.ZETp9Cscxi + return 0 + desc cleanup + set +o xtrace ----------------------------------------------------------------------------------- cleanup ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.ZcNiZeh0er ++ mktemp + local LAST_ERR=/tmp/tmp.cYlVTsq44F + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1737/e2e-tests/smart-update1/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZcNiZeh0er deployment.apps "version-service" deleted service "version-service" deleted + cat /tmp/tmp.cYlVTsq44F + rm /tmp/tmp.ZcNiZeh0er /tmp/tmp.cYlVTsq44F + return 0 + destroy smart-update1-19776 + local namespace=smart-update1-19776 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v level=info + grep -v 'the object has been modified' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'get backup status: Job.batch' + sort -u ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ + tee /tmp/tmp.j6tvA4WuUZ/operator.log +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.yz9cSnvFUK +++ mktemp ++ local LAST_ERR=/tmp/tmp.8vhucfxOJa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yz9cSnvFUK ++ cat /tmp/tmp.8vhucfxOJa ++ rm /tmp/tmp.yz9cSnvFUK /tmp/tmp.8vhucfxOJa ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-6c9cbdddbf-qls6l ++ mktemp + local LAST_OUT=/tmp/tmp.xKIvS4uFRq ++ mktemp + local LAST_ERR=/tmp/tmp.WEBoTajxjB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-6c9cbdddbf-qls6l + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xKIvS4uFRq + cat /tmp/tmp.WEBoTajxjB + rm /tmp/tmp.xKIvS4uFRq /tmp/tmp.WEBoTajxjB + return 0 2024-07-05T09:19:37.636Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.26.15-gke.1469001"} 2024-07-05T09:19:37.637Z INFO setup Manager starting up {"gitCommit": "0ed7c6427f0e7f4e674956a926a011fcc8e46b98", "gitBranch": "PR-1737-0ed7c642", "buildTime": "2024-07-05T07:35:34Z", "goVersion": "go1.22.5", "os": "linux", "arch": "amd64"} 2024-07-05T09:19:37.638Z INFO setup Registering Components. 2024-07-05T09:19:39.925Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2024-07-05T09:19:39.928Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2024-07-05T09:19:39.928Z INFO controller-runtime.metrics Starting metrics server 2024-07-05T09:19:39.928Z INFO controller-runtime.webhook Starting webhook server 2024-07-05T09:19:39.928Z INFO setup Starting the Cmd. 2024-07-05T09:19:39.928Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2024-07-05T09:19:39.929Z INFO controller-runtime.certwatcher Starting certificate watcher 2024-07-05T09:19:39.929Z INFO controller-runtime.certwatcher Updated current TLS certificate 2024-07-05T09:19:39.929Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2024-07-05T09:19:40.107Z INFO attempting to acquire leader lease pxc-operator/08db1feb.percona.com... 2024-07-05T09:19:40.122Z INFO successfully acquired lease pxc-operator/08db1feb.percona.com 2024-07-05T09:19:40.123Z DEBUG events percona-xtradb-cluster-operator-6c9cbdddbf-qls6l_1362d9c3-3eef-4841-87f2-3c3925272dcb became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"bad2cca2-16c6-47b2-b157-73e1a46ac2ec","apiVersion":"coordination.k8s.io/v1","resourceVersion":"52751"}, "reason": "LeaderElection"} 2024-07-05T09:19:40.123Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2024-07-05T09:19:40.123Z INFO Starting Controller {"controller": "pxc-controller"} 2024-07-05T09:19:40.123Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2024-07-05T09:19:40.123Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2024-07-05T09:19:40.123Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"} 2024-07-05T09:19:40.123Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2024-07-05T09:19:40.230Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2024-07-05T09:19:40.230Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2024-07-05T09:19:40.237Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2024-07-05T09:22:12.743Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "94d9f51e-1187-4617-837d-2c59fe56565a"} 2024-07-05T09:22:13.694Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "aed27270-e805-4c6d-9e05-a486f38015e0"} 2024-07-05T09:22:14.052Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "a5858937-00c2-4321-b7b5-20f3497975eb"} 2024-07-05T09:22:19.028Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "b4fd20b1-8850-4e7d-9848-954b86d2cf53"} 2024-07-05T09:22:24.178Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "de2341a8-bf99-4525-90ce-e96e89ff879e"} 2024-07-05T09:22:29.329Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "88c7314d-8467-417b-88c4-8da0de62381b"} 2024-07-05T09:22:34.480Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "cd6ff08a-026c-4c3a-b319-3b13be62cad2"} 2024-07-05T09:22:34.645Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "add85f2d-4b4a-43da-8d1c-8632a3b63afa"} 2024-07-05T09:22:39.709Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4b49f3b1-4ef3-4d11-97bc-03d9424b0bad"} 2024-07-05T09:22:44.843Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "2d33b692-46be-4512-be83-9ebfff27151f"} 2024-07-05T09:22:50.004Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "8c2cfde3-f42d-4188-b913-d2ad3c719b5f"} 2024-07-05T09:23:27.473Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "d87381a8-c9e1-415b-82f3-e3a10bf7bfb4", "user": "operator"} 2024-07-05T09:23:27.507Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "d87381a8-c9e1-415b-82f3-e3a10bf7bfb4", "user": "monitor"} 2024-07-05T09:23:27.574Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "d87381a8-c9e1-415b-82f3-e3a10bf7bfb4"} 2024-07-05T09:23:27.609Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "d87381a8-c9e1-415b-82f3-e3a10bf7bfb4"} 2024-07-05T09:23:27.643Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "d87381a8-c9e1-415b-82f3-e3a10bf7bfb4", "user": "xtrabackup"} 2024-07-05T09:23:27.693Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "d87381a8-c9e1-415b-82f3-e3a10bf7bfb4"} 2024-07-05T09:23:27.726Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "d87381a8-c9e1-415b-82f3-e3a10bf7bfb4", "user": "replication"} 2024-07-05T09:23:28.810Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "d87381a8-c9e1-415b-82f3-e3a10bf7bfb4", "err": "get primary pxc pod: not found"} 2024-07-05T09:23:33.591Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "1839589a-8bfc-4f96-a877-6f2a3b92dcea", "err": "get primary pxc pod: not found"} 2024-07-05T09:23:40.005Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "79a63ccd-cdc1-47ef-a0d9-41ccb26a27bd", "err": "get primary pxc pod: not found"} 2024-07-05T09:23:46.168Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "03f81dd2-58b2-44aa-b8ef-7f2559fd9507", "err": "get primary pxc pod: not found"} 2024-07-05T09:25:57.764Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "3a2ee19f-2ae2-46be-a273-64ab3ac3a01c", "user": "root"} 2024-07-05T09:25:58.958Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "3a2ee19f-2ae2-46be-a273-64ab3ac3a01c", "new version": "8.0.19-10"} 2024-07-05T09:26:00.293Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "3a2ee19f-2ae2-46be-a273-64ab3ac3a01c"} 2024-07-05T09:26:05.082Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "61fc3b1d-29bc-4443-8b5e-392eb43a9e06"} 2024-07-05T09:26:11.481Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "3017b3f5-b8d3-49d0-ae88-aff51ab56b91"} 2024-07-05T09:26:17.669Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "8b6c0f46-4781-43d7-ae5d-f0c99febb31d"} 2024-07-05T09:26:23.962Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "b7b907e0-18ee-4cd3-b26c-4b27cd3063af"} 2024-07-05T09:26:30.233Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "15a220e0-6ac2-4efd-b154-141134a577a5"} 2024-07-05T09:26:36.835Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "09b928da-85d3-4fe4-b4cd-822ff3a06b93"} 2024-07-05T09:26:43.623Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "b6fc1e9e-9758-4b82-bd47-6979cf4c4307"} 2024-07-05T09:26:49.835Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "e0095a32-3ba6-49c6-99ee-6fd494cf4402"} 2024-07-05T09:26:56.335Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "2a4437bc-ce62-4b1d-bb21-c970c810b8ce"} 2024-07-05T09:27:02.617Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "8ff0a834-a9b2-48c9-8abc-bfa811d3801b"} 2024-07-05T09:27:06.974Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97"} 2024-07-05T09:27:07.010Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-2"} 2024-07-05T09:27:07.010Z INFO primary pod {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-0.smart-update-pxc.smart-update1-19776.svc.cluster.local"} 2024-07-05T09:27:08.372Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "670cf1a0-1f9e-4022-aa9d-4c857daae3e3"} 2024-07-05T09:27:27.252Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-2", "reason": "PodInitializing"} 2024-07-05T09:28:27.288Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-2"} 2024-07-05T09:29:27.327Z INFO pod present in hostgroups {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-2"} 2024-07-05T09:29:27.334Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-1"} 2024-07-05T09:29:27.334Z INFO pod is online {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-2"} 2024-07-05T09:29:47.560Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-1", "reason": "PodInitializing"} 2024-07-05T09:30:47.579Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-1"} 2024-07-05T09:31:47.602Z INFO pod present in hostgroups {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-1"} 2024-07-05T09:31:47.608Z INFO apply changes to primary pod {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-0"} 2024-07-05T09:31:47.608Z INFO pod is online {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-1"} 2024-07-05T09:32:07.829Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-0", "reason": "PodInitializing"} 2024-07-05T09:33:17.891Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-0"} 2024-07-05T09:34:27.915Z INFO pod present in hostgroups {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-0"} 2024-07-05T09:34:27.922Z INFO pod is online {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "pod name": "smart-update-pxc-0"} 2024-07-05T09:34:27.922Z INFO smart update finished {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97"} 2024-07-05T09:34:28.011Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4618a2be-58b0-4494-b1b0-bdd035034f97", "new version": "8.0.36-28.1"} 2024-07-05T09:34:30.847Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "82b0af33-df24-467f-836f-241d9370d8f9"} 2024-07-05T09:34:35.681Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "5a6ea0b4-a57b-4dcb-b2cd-825997aeeaf4"} 2024-07-05T09:34:42.321Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "c5d0771b-1f17-4be7-8ee5-631258b5822a"} 2024-07-05T09:34:48.898Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "c57d3005-a708-4b4e-8c1e-15325f377a01"} 2024-07-05T09:34:55.124Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "7135c526-269b-43f3-b89b-61a10794a4a2"} 2024-07-05T09:35:38.350Z ERROR Update status {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "1973ed32-0847-4c7f-b8b9-4a2efd9d6eec", "error": "check pxc upgrade progress: StatefulSet.apps \"smart-update-pxc\" not found", "errorVerbose": "StatefulSet.apps \"smart-update-pxc\" not found\ncheck pxc upgrade progress\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:130\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:201\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:245\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-05T09:35:54.973Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "27c279ff-9877-4309-8613-57c9af6d9faa"} 2024-07-05T09:35:56.462Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 6c4d0ff8-f669-4aed-9497-039e508b279c 2024-07-05T09:35:56.730Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "95b48fec-cbfc-4539-bc1d-52ca02c265fc"} 2024-07-05T09:35:58.448Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "ab6a5a2f-1ad1-478e-81fa-53f189ad010b"} 2024-07-05T09:36:03.598Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "43a91145-48a5-4de8-a8ee-0a22cc8f3b43"} 2024-07-05T09:36:08.740Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "c1ea6127-42a3-4498-be6e-ac65d437cdff"} 2024-07-05T09:36:13.942Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "11528eb1-c1f8-42af-9205-167ce17dc966"} 2024-07-05T09:36:19.078Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "3927bb3e-d9b1-44c7-a262-289cef59b15e"} 2024-07-05T09:36:24.221Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "2b2930f7-c433-4145-b261-efaa6a737af6"} 2024-07-05T09:36:29.344Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "dd9262fc-1d60-4aaf-861c-ff12f2d412e5"} 2024-07-05T09:36:34.480Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "f4372c61-7f3c-4281-a503-62e83f5c3d49"} 2024-07-05T09:36:39.689Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "38d89cec-1a79-4bef-add5-14e1d590a60e"} 2024-07-05T09:36:44.827Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "1953dd4d-6ac9-42c5-810b-f6e75b3dbc48"} 2024-07-05T09:36:49.987Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4b9c27c5-a028-43dc-b4bf-bc2ba831d43b"} 2024-07-05T09:36:55.119Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "1cddbadc-f401-4c00-a68e-709d39d859b5"} 2024-07-05T09:37:00.244Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "80e22ffc-2d9b-4d65-822e-d7df1eb19077"} 2024-07-05T09:37:05.373Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "a3f0128b-25d4-4087-af27-c9e85974c942"} 2024-07-05T09:37:10.636Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4a29cd64-57f7-4446-9679-5e14dcc9b1b0"} 2024-07-05T09:37:11.163Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "e48a3cce-fa5b-4604-b266-2c8498b403f4"} 2024-07-05T09:37:15.984Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "9251420e-115b-43d7-b7c9-fed7a491b9a6"} 2024-07-05T09:37:21.196Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4692ef1e-1a0e-47ca-9b5b-0d46805094a9"} 2024-07-05T09:37:26.391Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "c7b396ec-4547-41a2-bfff-b19c1c93c3a6"} 2024-07-05T09:37:31.593Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "9d9c63b5-0c60-412a-b7a3-5de9dcf4d786"} 2024-07-05T09:37:36.863Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "152ccb37-64de-4e22-9451-6763c78825d1"} 2024-07-05T09:39:37.649Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "c4e2f8be-dc2d-47d4-9044-8c94fcaab4a9", "new version": "8.0.19-10"} 2024-07-05T09:40:42.632Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "001a03f9-52b0-4909-a139-78ef8368d78f"} 2024-07-05T09:40:42.640Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "001a03f9-52b0-4909-a139-78ef8368d78f", "pod name": "smart-update-pxc-2"} 2024-07-05T09:40:42.640Z INFO primary pod {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "001a03f9-52b0-4909-a139-78ef8368d78f", "pod name": "smart-update-pxc-0.smart-update-pxc.smart-update1-19776"} 2024-07-05T09:41:02.983Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "001a03f9-52b0-4909-a139-78ef8368d78f", "pod name": "smart-update-pxc-2", "reason": "PodInitializing"} 2024-07-05T09:42:13.002Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "001a03f9-52b0-4909-a139-78ef8368d78f", "pod name": "smart-update-pxc-2"} 2024-07-05T09:42:13.016Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "001a03f9-52b0-4909-a139-78ef8368d78f", "pod name": "smart-update-pxc-1"} 2024-07-05T09:42:33.230Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "001a03f9-52b0-4909-a139-78ef8368d78f", "pod name": "smart-update-pxc-1", "reason": "PodInitializing"} 2024-07-05T09:43:43.265Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "001a03f9-52b0-4909-a139-78ef8368d78f", "pod name": "smart-update-pxc-1"} 2024-07-05T09:43:43.279Z INFO apply changes to primary pod {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "001a03f9-52b0-4909-a139-78ef8368d78f", "pod name": "smart-update-pxc-0"} 2024-07-05T09:45:03.515Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "001a03f9-52b0-4909-a139-78ef8368d78f", "pod name": "smart-update-pxc-0"} 2024-07-05T09:45:03.532Z INFO smart update finished {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "001a03f9-52b0-4909-a139-78ef8368d78f"} 2024-07-05T09:45:03.614Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "001a03f9-52b0-4909-a139-78ef8368d78f", "new version": "8.0.36-28.1"} 2024-07-05T09:45:55.132Z ERROR Update status {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "9866612c-0861-4af1-8514-046835a8d95c", "error": "write status: PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found", "errorVerbose": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found\nwrite status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:157\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:140\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:201\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:245\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-05T09:46:13.219Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "78435305-d923-40d4-ba56-b1229c002803"} 2024-07-05T09:46:13.929Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: b86c7a23-d40e-4f49-98f9-cc642696f3ac 2024-07-05T09:46:14.592Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4f2fc89c-e6ec-4db3-9aab-cfd70f65ca3a"} 2024-07-05T09:46:15.211Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "285e68d1-9701-44ed-b41d-1cb3dfa23c1e"} 2024-07-05T09:46:20.343Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "bc26db43-e75e-45ff-8e81-d456918d8466"} 2024-07-05T09:46:26.010Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "8011c809-48fc-4e25-85c3-be92395ab5fd"} 2024-07-05T09:46:31.271Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "85f2f300-c81c-4850-81b5-14eaae10b110"} 2024-07-05T09:46:36.490Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "8acdbf7b-a457-4fb2-b965-016ae88850c2"} 2024-07-05T09:46:41.621Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "c7e2e5db-3a72-420d-8a3f-0fc9c676192f"} 2024-07-05T09:46:46.747Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "4701c07c-0cf1-4125-9112-4443f3f06010"} 2024-07-05T09:46:51.870Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "e96d7824-c4e3-48fd-b09b-712c1b07e9f8"} 2024-07-05T09:46:57.056Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "38a99bf0-786b-4a8e-9206-802f27d45343"} 2024-07-05T09:47:02.170Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "373983b2-308e-4b82-9f1d-63b4a3b3930f"} 2024-07-05T09:47:07.326Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "88dc3b5c-67f7-4aff-be40-c7855e255720"} 2024-07-05T09:47:12.503Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "e6805dd1-822f-4316-bef1-aea689a30a91"} 2024-07-05T09:47:17.633Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "91e39ef2-f795-4909-a4f5-525c1bfa2bbe"} 2024-07-05T09:47:22.772Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "bca88975-8688-4d04-b6a5-1cfc1c27d626"} 2024-07-05T09:47:27.905Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "ae20e63b-d639-430d-b0ea-83ab956ce4bf"} 2024-07-05T09:47:33.060Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "d9f629fa-03f1-4764-99de-b5026bb714ac"} 2024-07-05T09:47:33.386Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "5b02f9c8-957d-4533-be6b-b60e826518cb"} 2024-07-05T09:47:34.716Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "86953a9d-0b79-4892-9a1f-7559ee6bf48d"} 2024-07-05T09:47:38.374Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "ce8622e3-1b1a-4fa4-af2f-cd0b5b44e52e"} 2024-07-05T09:47:43.614Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "69af5cac-c352-4491-912b-0d54e36ac04b"} 2024-07-05T09:47:48.805Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "8fe39f99-7fd8-4dc6-8475-5af045c67368"} 2024-07-05T09:47:54.080Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "f64b5898-81fb-47c7-9c4a-3924a3c33530"} 2024-07-05T09:47:59.419Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "9d236447-c566-40b0-966c-15e37b2acf9e"} 2024-07-05T09:49:59.833Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "85057684-ab25-455e-b3d4-bca60b5e2c9c", "new version": "8.0.19-10"} 2024-07-05T09:51:05.514Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "949d28bc-b576-44b8-87de-c152aad0408e"} 2024-07-05T09:51:05.526Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "949d28bc-b576-44b8-87de-c152aad0408e", "pod name": "smart-update-pxc-2"} 2024-07-05T09:51:05.526Z INFO primary pod {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "949d28bc-b576-44b8-87de-c152aad0408e", "pod name": "smart-update-pxc-0.smart-update-pxc.smart-update1-19776"} 2024-07-05T09:51:26.203Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "949d28bc-b576-44b8-87de-c152aad0408e", "pod name": "smart-update-pxc-2", "reason": "PodInitializing"} 2024-07-05T09:52:26.232Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "949d28bc-b576-44b8-87de-c152aad0408e", "pod name": "smart-update-pxc-2"} 2024-07-05T09:52:26.249Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "949d28bc-b576-44b8-87de-c152aad0408e", "pod name": "smart-update-pxc-1"} 2024-07-05T09:52:46.583Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "949d28bc-b576-44b8-87de-c152aad0408e", "pod name": "smart-update-pxc-1", "reason": "PodInitializing"} 2024-07-05T09:53:56.607Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "949d28bc-b576-44b8-87de-c152aad0408e", "pod name": "smart-update-pxc-1"} 2024-07-05T09:53:56.623Z INFO apply changes to primary pod {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "949d28bc-b576-44b8-87de-c152aad0408e", "pod name": "smart-update-pxc-0"} 2024-07-05T09:54:16.802Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "949d28bc-b576-44b8-87de-c152aad0408e", "pod name": "smart-update-pxc-0", "reason": "PodInitializing"} 2024-07-05T09:55:26.835Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "949d28bc-b576-44b8-87de-c152aad0408e", "pod name": "smart-update-pxc-0"} 2024-07-05T09:55:26.853Z INFO smart update finished {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "949d28bc-b576-44b8-87de-c152aad0408e"} 2024-07-05T09:55:26.931Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "949d28bc-b576-44b8-87de-c152aad0408e", "new version": "8.0.36-28.1"} 2024-07-05T09:56:25.019Z ERROR Update status {"controller": "pxc-controller", "namespace": "smart-update1-19776", "name": "smart-update", "reconcileID": "30bdb616-7f86-4a61-a34b-3db2410029e1", "error": "write status: PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found", "errorVerbose": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found\nwrite status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:157\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:140\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:201\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:245\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:203 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:245 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.v66Egtox2B ++ mktemp + local LAST_ERR=/tmp/tmp.cxvGW7TSGm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.v66Egtox2B No resources found + cat /tmp/tmp.cxvGW7TSGm + rm /tmp/tmp.v66Egtox2B /tmp/tmp.cxvGW7TSGm + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.IerSUlHp80 ++ mktemp + local LAST_ERR=/tmp/tmp.fQjJ1IjQlR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IerSUlHp80 No resources found + cat /tmp/tmp.fQjJ1IjQlR + rm /tmp/tmp.IerSUlHp80 /tmp/tmp.fQjJ1IjQlR + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.X9fOn3m03v ++ mktemp + local LAST_ERR=/tmp/tmp.6PRLoodMia + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.X9fOn3m03v No resources found + cat /tmp/tmp.6PRLoodMia + rm /tmp/tmp.X9fOn3m03v /tmp/tmp.6PRLoodMia + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.RYVhq6pfwT ++ mktemp + local LAST_ERR=/tmp/tmp.d9WCIgJEtA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RYVhq6pfwT validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.d9WCIgJEtA + rm /tmp/tmp.RYVhq6pfwT /tmp/tmp.d9WCIgJEtA + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace smart-update1-19776 + rm -rf /tmp/tmp.j6tvA4WuUZ ++ mktemp + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.pj6GeXOOBJ + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- ++ mktemp + local LAST_OUT=/tmp/tmp.DJBIoEJWba ++ mktemp + local LAST_ERR=/tmp/tmp.KDKhZuLm89 + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.Ej8AbLWLV8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace smart-update1-19776 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator