Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/logs/smart-update1-8-0.log WARNING: version difference between client (1.30) and server (1.27) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.30) and server (1.27) exceeds the supported minor version skew of +/-1 + API=pxc.percona.com/v9-9-9 + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + CLUSTER=smart-update + CLUSTER_SIZE=3 + PROXY_SIZE=2 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0 ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' + PXC_VER=8.0 + TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + VS_URL=http://version-service + VS_PORT=11000 + VS_ENDPOINT=http://version-service:11000 + main + create_infra smart-update1-3747 + local ns=smart-update1-3747 + '[' -n pxc-operator ']' + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n smart-update1-4037 smart-update --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/smart-update patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.v7bXzM6A3t ++ mktemp + local LAST_ERR=/tmp/tmp.xeB0JpGix4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.v7bXzM6A3t perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.xeB0JpGix4 + rm /tmp/tmp.v7bXzM6A3t /tmp/tmp.xeB0JpGix4 + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.qLRNA7t9db ++ mktemp + local LAST_ERR=/tmp/tmp.r4EksjkZ55 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qLRNA7t9db No resources found + cat /tmp/tmp.r4EksjkZ55 + rm /tmp/tmp.qLRNA7t9db /tmp/tmp.r4EksjkZ55 + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.X6JEwQPRba ++ mktemp + local LAST_ERR=/tmp/tmp.Je8OJ6sVSP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.X6JEwQPRba No resources found + cat /tmp/tmp.Je8OJ6sVSP + rm /tmp/tmp.X6JEwQPRba /tmp/tmp.Je8OJ6sVSP + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + xargs kubectl delete ns ++ mktemp ++ mktemp + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + local LAST_OUT=/tmp/tmp.wXnWyhAIyp ++ mktemp + local LAST_OUT=/tmp/tmp.uILA6lJL21 + local LAST_ERR=/tmp/tmp.JftUAqNgqC + local exit_status=0 ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.xjlmxB4CDO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + for i in '$(seq 0 2)' + kubectl delete namespace pxc-operator + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uILA6lJL21 + cat /tmp/tmp.xjlmxB4CDO + rm /tmp/tmp.uILA6lJL21 /tmp/tmp.xjlmxB4CDO + return 0 namespace "cert-manager" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted namespace "smart-update1-4037" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wXnWyhAIyp namespace "pxc-operator" deleted + cat /tmp/tmp.JftUAqNgqC + rm /tmp/tmp.wXnWyhAIyp /tmp/tmp.JftUAqNgqC + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'namespace/pxc-operator - ' namespace/pxc-operator - + set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.8jGRAJXO0v ++ mktemp + local LAST_ERR=/tmp/tmp.PzRexe2j0U + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8jGRAJXO0v namespace/pxc-operator created + cat /tmp/tmp.PzRexe2j0U + rm /tmp/tmp.8jGRAJXO0v /tmp/tmp.PzRexe2j0U + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.FaBGUcH5tM +++ mktemp ++ local LAST_ERR=/tmp/tmp.FQyKgoCmex ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FaBGUcH5tM ++ cat /tmp/tmp.FQyKgoCmex ++ rm /tmp/tmp.FaBGUcH5tM /tmp/tmp.FQyKgoCmex ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1753-50eda1a3-33-cluster8 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.pkHZDJwvp2 ++ mktemp + local LAST_ERR=/tmp/tmp.JZVI21ThPr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1753-50eda1a3-33-cluster8 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pkHZDJwvp2 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1753-50eda1a3-33-cluster8" modified. + cat /tmp/tmp.JZVI21ThPr + rm /tmp/tmp.pkHZDJwvp2 /tmp/tmp.JZVI21ThPr + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.6sqimrrJhE ++ mktemp + local LAST_ERR=/tmp/tmp.qUyzkiImc3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6sqimrrJhE customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.qUyzkiImc3 + rm /tmp/tmp.6sqimrrJhE /tmp/tmp.qUyzkiImc3 + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AezvvbYpQ7 ++ mktemp + local LAST_ERR=/tmp/tmp.gkRole3G5q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AezvvbYpQ7 clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.gkRole3G5q + rm /tmp/tmp.AezvvbYpQ7 /tmp/tmp.gkRole3G5q + return 0 + kubectl_bin apply -f - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/deploy/cw-operator.yaml + local LAST_OUT=/tmp/tmp.tlxJZ3ixYQ ++ mktemp + local LAST_ERR=/tmp/tmp.ONXOQPmPRO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tlxJZ3ixYQ deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.ONXOQPmPRO + rm /tmp/tmp.tlxJZ3ixYQ /tmp/tmp.ONXOQPmPRO + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.jrubQpIxJZ ++ mktemp + local LAST_ERR=/tmp/tmp.SGneVBt8zK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jrubQpIxJZ pod/percona-xtradb-cluster-operator-6f69b446fd-twpr8 condition met + cat /tmp/tmp.SGneVBt8zK + rm /tmp/tmp.jrubQpIxJZ /tmp/tmp.SGneVBt8zK + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.T1mBfx0pVh +++ mktemp ++ local LAST_ERR=/tmp/tmp.9bn1lKpgVM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.T1mBfx0pVh ++ cat /tmp/tmp.9bn1lKpgVM ++ rm /tmp/tmp.T1mBfx0pVh /tmp/tmp.9bn1lKpgVM ++ return 0 + wait_pod percona-xtradb-cluster-operator-6f69b446fd-twpr8 480 pxc-operator + local pod=percona-xtradb-cluster-operator-6f69b446fd-twpr8 + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-6f69b446fd-twpr8 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-6f69b446fd-twpr8 condition met percona-xtradb-cluster-operator-6f69b446fd-twpr8.Ok + sleep 3 + create_namespace smart-update1-3747 + local namespace=smart-update1-3747 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces smart-update1-3747' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces smart-update1-3747 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace smart-update1-3747 ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.F2CytWicBt ++ mktemp + local LAST_OUT=/tmp/tmp.AkqZt5sWpO ++ mktemp + local LAST_ERR=/tmp/tmp.o9dTezF1kS + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.5ov6tvYlAU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-3747 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-3747 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.F2CytWicBt + cat /tmp/tmp.o9dTezF1kS + rm /tmp/tmp.F2CytWicBt /tmp/tmp.o9dTezF1kS + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-3747 namespace "gmp-public" deleted namespace "gmp-system" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.AkqZt5sWpO + cat /tmp/tmp.5ov6tvYlAU Error from server (NotFound): namespaces "smart-update1-3747" not found + rm /tmp/tmp.AkqZt5sWpO /tmp/tmp.5ov6tvYlAU + return 1 + : + wait_for_delete namespace/smart-update1-3747 + local res=namespace/smart-update1-3747 + echo -n 'namespace/smart-update1-3747 - ' namespace/smart-update1-3747 - + set +o xtrace Error from server (NotFound): namespaces "smart-update1-3747" not found + desc 'create namespace smart-update1-3747' + set +o xtrace ----------------------------------------------------------------------------------- create namespace smart-update1-3747 ----------------------------------------------------------------------------------- + kubectl_bin create namespace smart-update1-3747 ++ mktemp + local LAST_OUT=/tmp/tmp.Y1xW2c8Nbx ++ mktemp + local LAST_ERR=/tmp/tmp.Jt9qvL3Pv2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace smart-update1-3747 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Y1xW2c8Nbx namespace/smart-update1-3747 created + cat /tmp/tmp.Jt9qvL3Pv2 + rm /tmp/tmp.Y1xW2c8Nbx /tmp/tmp.Jt9qvL3Pv2 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.N1U9MzpIa9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RB86lbv6VZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.N1U9MzpIa9 ++ cat /tmp/tmp.RB86lbv6VZ ++ rm /tmp/tmp.N1U9MzpIa9 /tmp/tmp.RB86lbv6VZ ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1753-50eda1a3-33-cluster8 --namespace=smart-update1-3747 ++ mktemp + local LAST_OUT=/tmp/tmp.lVt9AEcFVy ++ mktemp + local LAST_ERR=/tmp/tmp.hd7x0dL0Iu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1753-50eda1a3-33-cluster8 --namespace=smart-update1-3747 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lVt9AEcFVy Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1753-50eda1a3-33-cluster8" modified. + cat /tmp/tmp.hd7x0dL0Iu + rm /tmp/tmp.lVt9AEcFVy /tmp/tmp.hd7x0dL0Iu + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.fp3tZwAVUB ++ mktemp + local LAST_ERR=/tmp/tmp.N06n7k3iNg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fp3tZwAVUB secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.N06n7k3iNg + rm /tmp/tmp.fp3tZwAVUB /tmp/tmp.N06n7k3iNg + return 0 + deploy_version_service + desc 'install version service' + set +o xtrace ----------------------------------------------------------------------------------- install version service ----------------------------------------------------------------------------------- + kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json ++ mktemp + local LAST_OUT=/tmp/tmp.2332sF5zTF ++ mktemp + local LAST_ERR=/tmp/tmp.Ezekqt7CSP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2332sF5zTF configmap/versions created + cat /tmp/tmp.Ezekqt7CSP + rm /tmp/tmp.2332sF5zTF /tmp/tmp.Ezekqt7CSP + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.eeev8iA2CN ++ mktemp + local LAST_ERR=/tmp/tmp.7GkSy1y485 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eeev8iA2CN deployment.apps/version-service created service/version-service created + cat /tmp/tmp.7GkSy1y485 + rm /tmp/tmp.eeev8iA2CN /tmp/tmp.7GkSy1y485 + return 0 + sleep 10 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.2jQ1d9dvz5 ++ mktemp + local LAST_ERR=/tmp/tmp.GBhR2PST5T + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2jQ1d9dvz5 namespace/cert-manager created + cat /tmp/tmp.GBhR2PST5T + rm /tmp/tmp.2jQ1d9dvz5 /tmp/tmp.GBhR2PST5T + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.bc6jbXYFaR ++ mktemp + local LAST_ERR=/tmp/tmp.IuEwK3IpmT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bc6jbXYFaR namespace/cert-manager labeled + cat /tmp/tmp.IuEwK3IpmT + rm /tmp/tmp.bc6jbXYFaR /tmp/tmp.IuEwK3IpmT + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.r6GjaIyMw2 ++ mktemp + local LAST_ERR=/tmp/tmp.Z7x7D3w3Kg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.r6GjaIyMw2 namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews configured role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection configured rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.Z7x7D3w3Kg Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.r6GjaIyMw2 /tmp/tmp.Z7x7D3w3Kg + return 0 + '[' '' == 4.10 ']' + sleep 70 ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ grep :8.0 +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++ tail -n3 ++ sort -V ++ head -n1 ++++ grep -c percona-xtradb-cluster-operator ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++++ mktemp +++ local LAST_OUT=/tmp/tmp.IsokYQx05c ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NDwkB7mjHY +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.IsokYQx05c +++ cat /tmp/tmp.NDwkB7mjHY +++ rm /tmp/tmp.IsokYQx05c /tmp/tmp.NDwkB7mjHY +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-6f69b446fd-twpr8 -n pxc-operator -- curl -s http://version-service.smart-update1-3747.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.eUGPoeXKuA +++ mktemp ++ local LAST_ERR=/tmp/tmp.AFpSQGbyZx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-6f69b446fd-twpr8 -n pxc-operator -- curl -s http://version-service.smart-update1-3747.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eUGPoeXKuA ++ cat /tmp/tmp.AFpSQGbyZx Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.eUGPoeXKuA /tmp/tmp.AFpSQGbyZx ++ return 0 + IMAGE_PXC=percona/percona-xtradb-cluster:8.0.19-10.1 + desc 'patch crd' + set +o xtrace ----------------------------------------------------------------------------------- patch crd ----------------------------------------------------------------------------------- + kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' ++ mktemp + local LAST_OUT=/tmp/tmp.31sTMqY5rc ++ mktemp + local LAST_ERR=/tmp/tmp.pjvc8CbUar + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.31sTMqY5rc customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched + cat /tmp/tmp.pjvc8CbUar + rm /tmp/tmp.31sTMqY5rc /tmp/tmp.pjvc8CbUar + return 0 + desc 'Updating ProxySQL PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- Updating ProxySQL PXC cluster ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/conf/smart-update.yml /tmp/tmp.eDPRkBgipg/smart-update.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3"' /tmp/tmp.eDPRkBgipg/smart-update.yml + spinup_pxc smart-update /tmp/tmp.eDPRkBgipg/smart-update.yml + local cluster=smart-update + local config=/tmp/tmp.eDPRkBgipg/smart-update.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.15wJnKBRSJ ++ mktemp + local LAST_ERR=/tmp/tmp.uJk9BvQ7TG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.15wJnKBRSJ secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.uJk9BvQ7TG + rm /tmp/tmp.15wJnKBRSJ /tmp/tmp.uJk9BvQ7TG + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.D00eN71JBy + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-3747~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_ERR=/tmp/tmp.piCNUEBWo8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.D00eN71JBy deployment.apps/pxc-client created + cat /tmp/tmp.piCNUEBWo8 + rm /tmp/tmp.D00eN71JBy /tmp/tmp.piCNUEBWo8 + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.eDPRkBgipg/smart-update.yml + '[' -z '' ']' + cat_config /tmp/tmp.eDPRkBgipg/smart-update.yml + kubectl_bin apply -f - + cat /tmp/tmp.eDPRkBgipg/smart-update.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + local LAST_OUT=/tmp/tmp.HEdDq8yLyo + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-3747~ + local LAST_ERR=/tmp/tmp.A05hYASA9R + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HEdDq8yLyo perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.A05hYASA9R + rm /tmp/tmp.HEdDq8yLyo /tmp/tmp.A05hYASA9R + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VGjviIfr2m ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tF0ZHqsYPP +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.VGjviIfr2m +++ cat /tmp/tmp.tF0ZHqsYPP +++ rm /tmp/tmp.VGjviIfr2m /tmp/tmp.tF0ZHqsYPP +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lwJj6j1QnJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jYcXUe0CdD +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.lwJj6j1QnJ +++ cat /tmp/tmp.jYcXUe0CdD +++ rm /tmp/tmp.lwJj6j1QnJ /tmp/tmp.jYcXUe0CdD +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-proxysql ++ return + local proxy=smart-update-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3747 ++ mktemp + local LAST_OUT=/tmp/tmp.cefaGGPUlC ++ mktemp + local LAST_ERR=/tmp/tmp.AQ8Jti6K9i + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3747 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3747 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3747 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.cefaGGPUlC + cat /tmp/tmp.AQ8Jti6K9i error: no matching resources found + rm /tmp/tmp.cefaGGPUlC /tmp/tmp.AQ8Jti6K9i + return 1 + true + wait_for_running smart-update-proxysql 1 + local name=smart-update-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-proxysql-0 480 + local pod=smart-update-proxysql-0 + local max_retry=480 + local ns= ++ echo smart-update-proxysql-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace Error from server (NotFound): pods "smart-update-proxysql-0" not found smart-update-proxysql-0.........Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met smart-update-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met smart-update-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-proxysql -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PZSdeciI4j +++ mktemp ++ local LAST_ERR=/tmp/tmp.3M1yQ5Um8A ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PZSdeciI4j ++ cat /tmp/tmp.3M1yQ5Um8A ++ rm /tmp/tmp.PZSdeciI4j /tmp/tmp.3M1yQ5Um8A ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-proxysql -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.j70mblclwv +++ mktemp ++ local LAST_ERR=/tmp/tmp.b6vCs2NMFP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.j70mblclwv ++ cat /tmp/tmp.b6vCs2NMFP ++ rm /tmp/tmp.j70mblclwv /tmp/tmp.b6vCs2NMFP ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pzxjnt98i4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FgFS2vN5AY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pzxjnt98i4 ++ cat /tmp/tmp.FgFS2vN5AY ++ rm /tmp/tmp.pzxjnt98i4 /tmp/tmp.FgFS2vN5AY ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KUN3Vn4xSB +++ mktemp ++ local LAST_ERR=/tmp/tmp.8xjbIf4gXP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KUN3Vn4xSB ++ cat /tmp/tmp.8xjbIf4gXP ++ rm /tmp/tmp.KUN3Vn4xSB /tmp/tmp.8xjbIf4gXP ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OhdfZKM3Ym +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ax1uykveef ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OhdfZKM3Ym ++ cat /tmp/tmp.Ax1uykveef ++ rm /tmp/tmp.OhdfZKM3Ym /tmp/tmp.Ax1uykveef ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aW5iftW8Gv +++ mktemp ++ local LAST_ERR=/tmp/tmp.AAsG6o0lrC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aW5iftW8Gv ++ cat /tmp/tmp.AAsG6o0lrC Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.aW5iftW8Gv /tmp/tmp.AAsG6o0lrC ++ return 0 + '[' '' ']' +++ get_proxy smart-update +++ local target_cluster=smart-update ++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.WpnQsFVTys +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.uuLOUmpnxA ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.WpnQsFVTys ++++ cat /tmp/tmp.uuLOUmpnxA ++++ rm /tmp/tmp.WpnQsFVTys /tmp/tmp.uuLOUmpnxA ++++ return 0 +++ [[ '' == \t\r\u\e ]] ++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.udFYQanDRm +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.WeuE53oCkC ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.udFYQanDRm ++++ cat /tmp/tmp.WeuE53oCkC ++++ rm /tmp/tmp.udFYQanDRm /tmp/tmp.WeuE53oCkC ++++ return 0 +++ [[ true == \t\r\u\e ]] +++ echo smart-update-proxysql +++ return ++ get_proxy_primary '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=smart-update-proxysql-0 +++ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 proxysql +++ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' +++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' +++ local pod=smart-update-proxysql-0 +++ local container_name=proxysql +++ set +o xtrace ++ local ip=smart-update-pxc-0.smart-update-pxc.smart-update1-3747.svc.cluster.local +++ echo smart-update-pxc-0.smart-update-pxc.smart-update1-3747.svc.cluster.local +++ wc -l ++ '[' 1 '!=' 1 ']' ++ echo smart-update-pxc-0.smart-update-pxc.smart-update1-3747.svc.cluster.local ++ cut -d. -f1 + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.TjStucU2Y6 ++ mktemp + local LAST_ERR=/tmp/tmp.H4GssmIijq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TjStucU2Y6 perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.H4GssmIijq + rm /tmp/tmp.TjStucU2Y6 /tmp/tmp.H4GssmIijq + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update................................................................ smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.r8AdveaTLp +++ mktemp ++ local LAST_ERR=/tmp/tmp.o4u8r25yPh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.r8AdveaTLp ++ cat /tmp/tmp.o4u8r25yPh ++ rm /tmp/tmp.r8AdveaTLp /tmp/tmp.o4u8r25yPh ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VoyFfyHYXo +++ mktemp ++ local LAST_ERR=/tmp/tmp.xBEfl7OYo2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VoyFfyHYXo ++ cat /tmp/tmp.xBEfl7OYo2 ++ rm /tmp/tmp.VoyFfyHYXo /tmp/tmp.xBEfl7OYo2 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CTAQ9XSqWM +++ mktemp ++ local LAST_ERR=/tmp/tmp.vhTUh8CfKP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CTAQ9XSqWM ++ cat /tmp/tmp.vhTUh8CfKP ++ rm /tmp/tmp.CTAQ9XSqWM /tmp/tmp.vhTUh8CfKP ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LN1vkMDrbI +++ mktemp ++ local LAST_ERR=/tmp/tmp.HCSk1nH7Wt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LN1vkMDrbI ++ cat /tmp/tmp.HCSk1nH7Wt ++ rm /tmp/tmp.LN1vkMDrbI /tmp/tmp.HCSk1nH7Wt ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sRX4YPemwO +++ mktemp ++ local LAST_ERR=/tmp/tmp.gE8TZr3ze5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sRX4YPemwO ++ cat /tmp/tmp.gE8TZr3ze5 ++ rm /tmp/tmp.sRX4YPemwO /tmp/tmp.gE8TZr3ze5 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8mk02iVRU5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.fTK65FfUYk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8mk02iVRU5 ++ cat /tmp/tmp.fTK65FfUYk ++ rm /tmp/tmp.8mk02iVRU5 /tmp/tmp.fTK65FfUYk ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xO1t7zjhVo +++ mktemp ++ local LAST_ERR=/tmp/tmp.MTVf0MUgYT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xO1t7zjhVo ++ cat /tmp/tmp.MTVf0MUgYT ++ rm /tmp/tmp.xO1t7zjhVo /tmp/tmp.MTVf0MUgYT ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fD5nTX6LTp +++ mktemp ++ local LAST_ERR=/tmp/tmp.zeqANVxUtH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fD5nTX6LTp ++ cat /tmp/tmp.zeqANVxUtH ++ rm /tmp/tmp.fD5nTX6LTp /tmp/tmp.zeqANVxUtH ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wCBzrIzvoP +++ mktemp ++ local LAST_ERR=/tmp/tmp.sL0IMrRGQJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wCBzrIzvoP ++ cat /tmp/tmp.sL0IMrRGQJ ++ rm /tmp/tmp.wCBzrIzvoP /tmp/tmp.sL0IMrRGQJ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 8 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YefHIDJhWY +++ mktemp ++ local LAST_ERR=/tmp/tmp.A1k7WqDhsD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YefHIDJhWY ++ cat /tmp/tmp.A1k7WqDhsD ++ rm /tmp/tmp.YefHIDJhWY /tmp/tmp.A1k7WqDhsD ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 9 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pUhvUzPcqO +++ mktemp ++ local LAST_ERR=/tmp/tmp.sctyI4EpZC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pUhvUzPcqO ++ cat /tmp/tmp.sctyI4EpZC ++ rm /tmp/tmp.pUhvUzPcqO /tmp/tmp.sctyI4EpZC ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 10 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vt2BBHpVJr +++ mktemp ++ local LAST_ERR=/tmp/tmp.J9Btjh7OkE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vt2BBHpVJr ++ cat /tmp/tmp.J9Btjh7OkE ++ rm /tmp/tmp.vt2BBHpVJr /tmp/tmp.J9Btjh7OkE ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 11 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1HO0sfbaJC +++ mktemp ++ local LAST_ERR=/tmp/tmp.HU9iupDMLu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1HO0sfbaJC ++ cat /tmp/tmp.HU9iupDMLu ++ rm /tmp/tmp.1HO0sfbaJC /tmp/tmp.HU9iupDMLu ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 12 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TdleIqowEc +++ mktemp ++ local LAST_ERR=/tmp/tmp.X6W94KVull ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TdleIqowEc ++ cat /tmp/tmp.X6W94KVull ++ rm /tmp/tmp.TdleIqowEc /tmp/tmp.X6W94KVull ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 13 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HXh1QYk20w +++ mktemp ++ local LAST_ERR=/tmp/tmp.VqKB3ly9xX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HXh1QYk20w ++ cat /tmp/tmp.VqKB3ly9xX ++ rm /tmp/tmp.HXh1QYk20w /tmp/tmp.VqKB3ly9xX ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xL7NWEwmhE +++ mktemp ++ local LAST_ERR=/tmp/tmp.01hr1StYBA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xL7NWEwmhE ++ cat /tmp/tmp.01hr1StYBA ++ rm /tmp/tmp.xL7NWEwmhE /tmp/tmp.01hr1StYBA ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.LZzKIrpo7V ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.v6qFJwXx5Z +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.LZzKIrpo7V +++++ cat /tmp/tmp.v6qFJwXx5Z +++++ rm /tmp/tmp.LZzKIrpo7V /tmp/tmp.v6qFJwXx5Z +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.bhBIUOGg2a ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.w4fnGZUKl8 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.bhBIUOGg2a +++++ cat /tmp/tmp.w4fnGZUKl8 +++++ rm /tmp/tmp.bhBIUOGg2a /tmp/tmp.w4fnGZUKl8 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-proxysql ++++ return +++ local cluster_proxy=smart-update-proxysql +++ echo proxysql ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RTjri1PbRB +++ mktemp ++ local LAST_ERR=/tmp/tmp.8lhTK5awrR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RTjri1PbRB ++ cat /tmp/tmp.8lhTK5awrR ++ rm /tmp/tmp.RTjri1PbRB /tmp/tmp.8lhTK5awrR ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UTVunxpnj8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Aa6ktY4bSk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UTVunxpnj8 ++ cat /tmp/tmp.Aa6ktY4bSk ++ rm /tmp/tmp.UTVunxpnj8 /tmp/tmp.Aa6ktY4bSk ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3BxU8sFKyd +++ mktemp ++ local LAST_ERR=/tmp/tmp.YVuj7PlODS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3BxU8sFKyd ++ cat /tmp/tmp.YVuj7PlODS ++ rm /tmp/tmp.3BxU8sFKyd /tmp/tmp.YVuj7PlODS ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.05O4mgWEpG +++ mktemp ++ local LAST_ERR=/tmp/tmp.YHHlpPhq4p ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.05O4mgWEpG ++ cat /tmp/tmp.YHHlpPhq4p ++ rm /tmp/tmp.05O4mgWEpG /tmp/tmp.YHHlpPhq4p ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/conf/smart-update.yml ++ mktemp + local LAST_OUT=/tmp/tmp.MBVpQctB70 ++ mktemp + local LAST_ERR=/tmp/tmp.rpde5UwkQU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/conf/smart-update.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MBVpQctB70 perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.rpde5UwkQU + rm /tmp/tmp.MBVpQctB70 /tmp/tmp.rpde5UwkQU + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.SQzwHzM1fQ ++ mktemp + local LAST_ERR=/tmp/tmp.dNXQt6scat + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SQzwHzM1fQ persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted persistentvolumeclaim "proxydata-smart-update-proxysql-0" deleted persistentvolumeclaim "proxydata-smart-update-proxysql-1" deleted + cat /tmp/tmp.dNXQt6scat + rm /tmp/tmp.SQzwHzM1fQ /tmp/tmp.dNXQt6scat + return 0 + desc 'Updating HAProxy PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- Updating HAProxy PXC cluster ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/conf/smart-update-haproxy.yml /tmp/tmp.eDPRkBgipg/smart-update-haproxy.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3"' /tmp/tmp.eDPRkBgipg/smart-update-haproxy.yml + spinup_pxc smart-update /tmp/tmp.eDPRkBgipg/smart-update-haproxy.yml + local cluster=smart-update + local config=/tmp/tmp.eDPRkBgipg/smart-update-haproxy.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.MUQnBMfhlq ++ mktemp + local LAST_ERR=/tmp/tmp.hRhlSnnmoV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MUQnBMfhlq secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.hRhlSnnmoV + rm /tmp/tmp.MUQnBMfhlq /tmp/tmp.hRhlSnnmoV + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_OUT=/tmp/tmp.qLWMVRgy18 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-3747~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.HlNV9FrOaK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qLWMVRgy18 deployment.apps/pxc-client unchanged + cat /tmp/tmp.HlNV9FrOaK + rm /tmp/tmp.qLWMVRgy18 /tmp/tmp.HlNV9FrOaK + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.eDPRkBgipg/smart-update-haproxy.yml + '[' -z '' ']' + cat_config /tmp/tmp.eDPRkBgipg/smart-update-haproxy.yml + kubectl_bin apply -f - + cat /tmp/tmp.eDPRkBgipg/smart-update-haproxy.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + local LAST_OUT=/tmp/tmp.1T6T3TkwD3 + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-3747~ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' ++ mktemp + local LAST_ERR=/tmp/tmp.pJv6HXSitz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1T6T3TkwD3 perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.pJv6HXSitz + rm /tmp/tmp.1T6T3TkwD3 /tmp/tmp.pJv6HXSitz + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.IEffz4oB1x ++++ mktemp +++ local LAST_ERR=/tmp/tmp.btYqDJj2n3 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.IEffz4oB1x +++ cat /tmp/tmp.btYqDJj2n3 +++ rm /tmp/tmp.IEffz4oB1x /tmp/tmp.btYqDJj2n3 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3747 ++ mktemp + local LAST_OUT=/tmp/tmp.JpgqCrj2eY ++ mktemp + local LAST_ERR=/tmp/tmp.f5M3KgLSQJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3747 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3747 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3747 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.JpgqCrj2eY + cat /tmp/tmp.f5M3KgLSQJ error: no matching resources found + rm /tmp/tmp.JpgqCrj2eY /tmp/tmp.f5M3KgLSQJ + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met smart-update-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init), haproxy-init (init) .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met smart-update-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met smart-update-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2mXyohpwRE +++ mktemp ++ local LAST_ERR=/tmp/tmp.ijxjvFprQQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2mXyohpwRE ++ cat /tmp/tmp.ijxjvFprQQ ++ rm /tmp/tmp.2mXyohpwRE /tmp/tmp.ijxjvFprQQ ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.90iwIYOmwe +++ mktemp ++ local LAST_ERR=/tmp/tmp.EGiVzwI8DX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.90iwIYOmwe ++ cat /tmp/tmp.EGiVzwI8DX ++ rm /tmp/tmp.90iwIYOmwe /tmp/tmp.EGiVzwI8DX ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MpidOhCoLS +++ mktemp ++ local LAST_ERR=/tmp/tmp.8o0P9ysqo1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MpidOhCoLS ++ cat /tmp/tmp.8o0P9ysqo1 ++ rm /tmp/tmp.MpidOhCoLS /tmp/tmp.8o0P9ysqo1 ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OZJyKyj4xl +++ mktemp ++ local LAST_ERR=/tmp/tmp.pR1dRTHoRp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OZJyKyj4xl ++ cat /tmp/tmp.pR1dRTHoRp ++ rm /tmp/tmp.OZJyKyj4xl /tmp/tmp.pR1dRTHoRp ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.redsL5QAuA +++ mktemp ++ local LAST_ERR=/tmp/tmp.ckYSW92kVB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.redsL5QAuA ++ cat /tmp/tmp.ckYSW92kVB ++ rm /tmp/tmp.redsL5QAuA /tmp/tmp.ckYSW92kVB ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XqH6EPcwOR +++ mktemp ++ local LAST_ERR=/tmp/tmp.sY5r3AZcKR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XqH6EPcwOR ++ cat /tmp/tmp.sY5r3AZcKR Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.XqH6EPcwOR /tmp/tmp.sY5r3AZcKR ++ return 0 + '[' '' ']' ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.1MNTnV45gP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LalfPhzaAB +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.1MNTnV45gP +++ cat /tmp/tmp.LalfPhzaAB +++ rm /tmp/tmp.1MNTnV45gP /tmp/tmp.LalfPhzaAB +++ return 0 ++ client_pod=pxc-client-65c795cbdf-xjdqv ++ wait_pod pxc-client-65c795cbdf-xjdqv ++ local pod=pxc-client-65c795cbdf-xjdqv ++ local max_retry=480 ++ local ns= +++ echo pxc-client-65c795cbdf-xjdqv +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.HBop8kFxsv ++ mktemp + local LAST_ERR=/tmp/tmp.g33u7T5aBy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HBop8kFxsv perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.g33u7T5aBy + rm /tmp/tmp.HBop8kFxsv /tmp/tmp.g33u7T5aBy + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update......................................... smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dotv89jByV +++ mktemp ++ local LAST_ERR=/tmp/tmp.CU1ydaM5QK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dotv89jByV ++ cat /tmp/tmp.CU1ydaM5QK ++ rm /tmp/tmp.dotv89jByV /tmp/tmp.CU1ydaM5QK ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KcyQqO34kg +++ mktemp ++ local LAST_ERR=/tmp/tmp.b2mp6tekQp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KcyQqO34kg ++ cat /tmp/tmp.b2mp6tekQp ++ rm /tmp/tmp.KcyQqO34kg /tmp/tmp.b2mp6tekQp ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5gcg7xsp5F +++ mktemp ++ local LAST_ERR=/tmp/tmp.tZ4yrj9aBw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5gcg7xsp5F ++ cat /tmp/tmp.tZ4yrj9aBw ++ rm /tmp/tmp.5gcg7xsp5F /tmp/tmp.tZ4yrj9aBw ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ww3ncccm8u +++ mktemp ++ local LAST_ERR=/tmp/tmp.h5S6fgghPR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ww3ncccm8u ++ cat /tmp/tmp.h5S6fgghPR ++ rm /tmp/tmp.Ww3ncccm8u /tmp/tmp.h5S6fgghPR ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1lPcu4Axqt +++ mktemp ++ local LAST_ERR=/tmp/tmp.Pj13gmas2W ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1lPcu4Axqt ++ cat /tmp/tmp.Pj13gmas2W ++ rm /tmp/tmp.1lPcu4Axqt /tmp/tmp.Pj13gmas2W ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xVLFY5yioy +++ mktemp ++ local LAST_ERR=/tmp/tmp.rojyOySBVL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xVLFY5yioy ++ cat /tmp/tmp.rojyOySBVL ++ rm /tmp/tmp.xVLFY5yioy /tmp/tmp.rojyOySBVL ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pWFTWmqvcI +++ mktemp ++ local LAST_ERR=/tmp/tmp.s3sBboaRfN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pWFTWmqvcI ++ cat /tmp/tmp.s3sBboaRfN ++ rm /tmp/tmp.pWFTWmqvcI /tmp/tmp.s3sBboaRfN ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sawSmgHjId +++ mktemp ++ local LAST_ERR=/tmp/tmp.MVrvMFYP51 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sawSmgHjId ++ cat /tmp/tmp.MVrvMFYP51 ++ rm /tmp/tmp.sawSmgHjId /tmp/tmp.MVrvMFYP51 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BkN0nj60HR +++ mktemp ++ local LAST_ERR=/tmp/tmp.6Mqu2cD3Mg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BkN0nj60HR ++ cat /tmp/tmp.6Mqu2cD3Mg ++ rm /tmp/tmp.BkN0nj60HR /tmp/tmp.6Mqu2cD3Mg ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WNnSItX3n4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.WgRZluk1Ye ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WNnSItX3n4 ++ cat /tmp/tmp.WgRZluk1Ye ++ rm /tmp/tmp.WNnSItX3n4 /tmp/tmp.WgRZluk1Ye ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.mJhDtnzwkN ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.dE9wYccd8a +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.mJhDtnzwkN +++++ cat /tmp/tmp.dE9wYccd8a +++++ rm /tmp/tmp.mJhDtnzwkN /tmp/tmp.dE9wYccd8a +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bTS44v04dp +++ mktemp ++ local LAST_ERR=/tmp/tmp.QegRUno6mA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bTS44v04dp ++ cat /tmp/tmp.QegRUno6mA ++ rm /tmp/tmp.bTS44v04dp /tmp/tmp.QegRUno6mA ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DAkq5mBbLy +++ mktemp ++ local LAST_ERR=/tmp/tmp.KQPh8bHV2w ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DAkq5mBbLy ++ cat /tmp/tmp.KQPh8bHV2w ++ rm /tmp/tmp.DAkq5mBbLy /tmp/tmp.KQPh8bHV2w ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6xxTxj25Sb +++ mktemp ++ local LAST_ERR=/tmp/tmp.CeEtUzieRU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6xxTxj25Sb ++ cat /tmp/tmp.CeEtUzieRU ++ rm /tmp/tmp.6xxTxj25Sb /tmp/tmp.CeEtUzieRU ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qtbtTABaRu +++ mktemp ++ local LAST_ERR=/tmp/tmp.oKnpRWNvXs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qtbtTABaRu ++ cat /tmp/tmp.oKnpRWNvXs ++ rm /tmp/tmp.qtbtTABaRu /tmp/tmp.oKnpRWNvXs ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + kubectl_bin delete -f /tmp/tmp.eDPRkBgipg/smart-update-haproxy.yml ++ mktemp + local LAST_OUT=/tmp/tmp.03Vx41ZQET ++ mktemp + local LAST_ERR=/tmp/tmp.Wn15qvlx4U + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.eDPRkBgipg/smart-update-haproxy.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.03Vx41ZQET perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.Wn15qvlx4U + rm /tmp/tmp.03Vx41ZQET /tmp/tmp.Wn15qvlx4U + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.6xzLvwRgBj ++ mktemp + local LAST_ERR=/tmp/tmp.q0L8623p5z + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6xzLvwRgBj persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.q0L8623p5z + rm /tmp/tmp.6xzLvwRgBj /tmp/tmp.q0L8623p5z + return 0 + desc 'Updating PXC cluster with version service available but disabled' + set +o xtrace ----------------------------------------------------------------------------------- Updating PXC cluster with version service available but disabled ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/conf/smart-update-version-service-reachable.yml /tmp/tmp.eDPRkBgipg/smart-update-version-service-reachable.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3"' /tmp/tmp.eDPRkBgipg/smart-update-version-service-reachable.yml + spinup_pxc smart-update /tmp/tmp.eDPRkBgipg/smart-update-version-service-reachable.yml + local cluster=smart-update + local config=/tmp/tmp.eDPRkBgipg/smart-update-version-service-reachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xIJa7rzA0T ++ mktemp + local LAST_ERR=/tmp/tmp.bkKuhHuOId + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xIJa7rzA0T secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.bkKuhHuOId + rm /tmp/tmp.xIJa7rzA0T /tmp/tmp.bkKuhHuOId + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.kXNyOuxxVF + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-3747~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_ERR=/tmp/tmp.LPl5ZngBk1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kXNyOuxxVF deployment.apps/pxc-client unchanged + cat /tmp/tmp.LPl5ZngBk1 + rm /tmp/tmp.kXNyOuxxVF /tmp/tmp.LPl5ZngBk1 + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.eDPRkBgipg/smart-update-version-service-reachable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.eDPRkBgipg/smart-update-version-service-reachable.yml + cat /tmp/tmp.eDPRkBgipg/smart-update-version-service-reachable.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + local LAST_OUT=/tmp/tmp.XQoQpD7CYJ + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-3747~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3#' + local LAST_ERR=/tmp/tmp.soGDGlaa9B + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XQoQpD7CYJ perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.soGDGlaa9B + rm /tmp/tmp.XQoQpD7CYJ /tmp/tmp.soGDGlaa9B + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rg6BhE9qyk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HXTTffcxL0 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.rg6BhE9qyk +++ cat /tmp/tmp.HXTTffcxL0 +++ rm /tmp/tmp.rg6BhE9qyk /tmp/tmp.HXTTffcxL0 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3747 ++ mktemp + local LAST_OUT=/tmp/tmp.Hcd8nKH9Wh ++ mktemp + local LAST_ERR=/tmp/tmp.yBiDfTgVOG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3747 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3747 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3747 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.Hcd8nKH9Wh + cat /tmp/tmp.yBiDfTgVOG error: no matching resources found + rm /tmp/tmp.Hcd8nKH9Wh /tmp/tmp.yBiDfTgVOG + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met smart-update-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init), haproxy-init (init) .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met smart-update-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met smart-update-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xA8JioHf4K +++ mktemp ++ local LAST_ERR=/tmp/tmp.XI91yubTzj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xA8JioHf4K ++ cat /tmp/tmp.XI91yubTzj ++ rm /tmp/tmp.xA8JioHf4K /tmp/tmp.XI91yubTzj ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1bTH2KVRT3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.J4txQphd2r ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1bTH2KVRT3 ++ cat /tmp/tmp.J4txQphd2r ++ rm /tmp/tmp.1bTH2KVRT3 /tmp/tmp.J4txQphd2r ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fl85gv4ABy +++ mktemp ++ local LAST_ERR=/tmp/tmp.6F2WcHjHgq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fl85gv4ABy ++ cat /tmp/tmp.6F2WcHjHgq ++ rm /tmp/tmp.fl85gv4ABy /tmp/tmp.6F2WcHjHgq ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZfwsLt6xEy +++ mktemp ++ local LAST_ERR=/tmp/tmp.tpZ6KpmQCJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZfwsLt6xEy ++ cat /tmp/tmp.tpZ6KpmQCJ ++ rm /tmp/tmp.ZfwsLt6xEy /tmp/tmp.tpZ6KpmQCJ ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZhqJQghFYz +++ mktemp ++ local LAST_ERR=/tmp/tmp.FwJdIUjQiA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZhqJQghFYz ++ cat /tmp/tmp.FwJdIUjQiA ++ rm /tmp/tmp.ZhqJQghFYz /tmp/tmp.FwJdIUjQiA ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UQ3m59u9tJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.c0i1ERboGx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UQ3m59u9tJ ++ cat /tmp/tmp.c0i1ERboGx Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.UQ3m59u9tJ /tmp/tmp.c0i1ERboGx ++ return 0 + '[' '' ']' ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6oeXE4QYez ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zEKKDmCGjd +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.6oeXE4QYez +++ cat /tmp/tmp.zEKKDmCGjd +++ rm /tmp/tmp.6oeXE4QYez /tmp/tmp.zEKKDmCGjd +++ return 0 ++ client_pod=pxc-client-65c795cbdf-xjdqv ++ wait_pod pxc-client-65c795cbdf-xjdqv ++ local pod=pxc-client-65c795cbdf-xjdqv ++ local max_retry=480 ++ local ns= +++ echo pxc-client-65c795cbdf-xjdqv +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.QLaYoFrRB0 ++ mktemp + local LAST_ERR=/tmp/tmp.mYrvWzdfNK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QLaYoFrRB0 perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.mYrvWzdfNK + rm /tmp/tmp.QLaYoFrRB0 /tmp/tmp.mYrvWzdfNK + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update................................................. smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZIaWCo2azj +++ mktemp ++ local LAST_ERR=/tmp/tmp.tM1n7BJo4O ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZIaWCo2azj ++ cat /tmp/tmp.tM1n7BJo4O ++ rm /tmp/tmp.ZIaWCo2azj /tmp/tmp.tM1n7BJo4O ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sQfgELFt6H +++ mktemp ++ local LAST_ERR=/tmp/tmp.dcRljH9rGp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sQfgELFt6H ++ cat /tmp/tmp.dcRljH9rGp ++ rm /tmp/tmp.sQfgELFt6H /tmp/tmp.dcRljH9rGp ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qrWehJeinR +++ mktemp ++ local LAST_ERR=/tmp/tmp.usstl8mBMZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qrWehJeinR ++ cat /tmp/tmp.usstl8mBMZ ++ rm /tmp/tmp.qrWehJeinR /tmp/tmp.usstl8mBMZ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cBCgeMeKoW +++ mktemp ++ local LAST_ERR=/tmp/tmp.u0nEIzS4vR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cBCgeMeKoW ++ cat /tmp/tmp.u0nEIzS4vR ++ rm /tmp/tmp.cBCgeMeKoW /tmp/tmp.u0nEIzS4vR ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.noCBYhlU6B +++ mktemp ++ local LAST_ERR=/tmp/tmp.vuzj0LbGFg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.noCBYhlU6B ++ cat /tmp/tmp.vuzj0LbGFg ++ rm /tmp/tmp.noCBYhlU6B /tmp/tmp.vuzj0LbGFg ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4tvoNBAyAF +++ mktemp ++ local LAST_ERR=/tmp/tmp.Bj5AINIrNM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4tvoNBAyAF ++ cat /tmp/tmp.Bj5AINIrNM ++ rm /tmp/tmp.4tvoNBAyAF /tmp/tmp.Bj5AINIrNM ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yGarsFiWd9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.UFVAeQOcD9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yGarsFiWd9 ++ cat /tmp/tmp.UFVAeQOcD9 ++ rm /tmp/tmp.yGarsFiWd9 /tmp/tmp.UFVAeQOcD9 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4MrQ2fMAjE +++ mktemp ++ local LAST_ERR=/tmp/tmp.32RjCslieC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4MrQ2fMAjE ++ cat /tmp/tmp.32RjCslieC ++ rm /tmp/tmp.4MrQ2fMAjE /tmp/tmp.32RjCslieC ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jacuhWkbJa +++ mktemp ++ local LAST_ERR=/tmp/tmp.1Q2RNcXKjP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jacuhWkbJa ++ cat /tmp/tmp.1Q2RNcXKjP ++ rm /tmp/tmp.jacuhWkbJa /tmp/tmp.1Q2RNcXKjP ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BViKfV32CM +++ mktemp ++ local LAST_ERR=/tmp/tmp.I0HhOKFAOe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BViKfV32CM ++ cat /tmp/tmp.I0HhOKFAOe ++ rm /tmp/tmp.BViKfV32CM /tmp/tmp.I0HhOKFAOe ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.C8gUPA6IQP ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.TAkKJA9g3g +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.C8gUPA6IQP +++++ cat /tmp/tmp.TAkKJA9g3g +++++ rm /tmp/tmp.C8gUPA6IQP /tmp/tmp.TAkKJA9g3g +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CKOAYLPaGJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.W3urjolzbU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CKOAYLPaGJ ++ cat /tmp/tmp.W3urjolzbU ++ rm /tmp/tmp.CKOAYLPaGJ /tmp/tmp.W3urjolzbU ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uboJZVGfz7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.6tGjSJ5xMl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uboJZVGfz7 ++ cat /tmp/tmp.6tGjSJ5xMl ++ rm /tmp/tmp.uboJZVGfz7 /tmp/tmp.6tGjSJ5xMl ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZanS372GGT +++ mktemp ++ local LAST_ERR=/tmp/tmp.YzpTwCxUlw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZanS372GGT ++ cat /tmp/tmp.YzpTwCxUlw ++ rm /tmp/tmp.ZanS372GGT /tmp/tmp.YzpTwCxUlw ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sHXgBvYrfA +++ mktemp ++ local LAST_ERR=/tmp/tmp.rDmW6EH7O9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sHXgBvYrfA ++ cat /tmp/tmp.rDmW6EH7O9 ++ rm /tmp/tmp.sHXgBvYrfA /tmp/tmp.rDmW6EH7O9 ++ return 0 + client_pod=pxc-client-65c795cbdf-xjdqv + wait_pod pxc-client-65c795cbdf-xjdqv + local pod=pxc-client-65c795cbdf-xjdqv + local max_retry=480 + local ns= ++ echo pxc-client-65c795cbdf-xjdqv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65c795cbdf-xjdqv condition met pxc-client-65c795cbdf-xjdqv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.eDPRkBgipg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.eDPRkBgipg/select-1.sql + kubectl_bin delete -f /tmp/tmp.eDPRkBgipg/smart-update-version-service-reachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.C1KNng0NA8 ++ mktemp + local LAST_ERR=/tmp/tmp.UfllWXbld6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.eDPRkBgipg/smart-update-version-service-reachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.C1KNng0NA8 perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.UfllWXbld6 + rm /tmp/tmp.C1KNng0NA8 /tmp/tmp.UfllWXbld6 + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.wgBmKgvdpt ++ mktemp + local LAST_ERR=/tmp/tmp.ozJy6mcx5D + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wgBmKgvdpt persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.ozJy6mcx5D + rm /tmp/tmp.wgBmKgvdpt /tmp/tmp.ozJy6mcx5D + return 0 + desc cleanup + set +o xtrace ----------------------------------------------------------------------------------- cleanup ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.aGg6H2LXuZ ++ mktemp + local LAST_ERR=/tmp/tmp.W5OQiInhx4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/smart-update1/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.aGg6H2LXuZ deployment.apps "version-service" deleted service "version-service" deleted + cat /tmp/tmp.W5OQiInhx4 + rm /tmp/tmp.aGg6H2LXuZ /tmp/tmp.W5OQiInhx4 + return 0 + destroy smart-update1-3747 + local namespace=smart-update1-3747 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v level=info + grep -v 'the object has been modified' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + tee /tmp/tmp.eDPRkBgipg/operator.log ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ + sort -u + grep -v 'get backup status: Job.batch' +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.2eUHIa8rfc +++ mktemp ++ local LAST_ERR=/tmp/tmp.CFg166bkvv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2eUHIa8rfc ++ cat /tmp/tmp.CFg166bkvv ++ rm /tmp/tmp.2eUHIa8rfc /tmp/tmp.CFg166bkvv ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-6f69b446fd-twpr8 ++ mktemp + local LAST_OUT=/tmp/tmp.yiHZyGCw7p ++ mktemp + local LAST_ERR=/tmp/tmp.kn1ak5u9W0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-6f69b446fd-twpr8 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yiHZyGCw7p + cat /tmp/tmp.kn1ak5u9W0 + rm /tmp/tmp.yiHZyGCw7p /tmp/tmp.kn1ak5u9W0 + return 0 2024-07-17T11:47:32.076Z INFO setup Manager starting up {"gitCommit": "50eda1a33a5de8689c5809cac616bd248a71cf6d", "gitBranch": "PR-1753-50eda1a3", "buildTime": "2024-07-17T09:55:54Z", "goVersion": "go1.22.5", "os": "linux", "arch": "amd64"} 2024-07-17T11:47:32.076Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.27.15-gke.1154000"} 2024-07-17T11:47:32.077Z INFO setup Registering Components. 2024-07-17T11:47:33.773Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2024-07-17T11:47:33.776Z INFO controller-runtime.metrics Starting metrics server 2024-07-17T11:47:33.776Z INFO setup Starting the Cmd. 2024-07-17T11:47:33.776Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2024-07-17T11:47:33.777Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2024-07-17T11:47:33.777Z INFO controller-runtime.webhook Starting webhook server 2024-07-17T11:47:33.778Z INFO controller-runtime.certwatcher Updated current TLS certificate 2024-07-17T11:47:33.779Z INFO controller-runtime.certwatcher Starting certificate watcher 2024-07-17T11:47:33.779Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2024-07-17T11:47:33.878Z INFO attempting to acquire leader lease pxc-operator/08db1feb.percona.com... 2024-07-17T11:47:33.897Z DEBUG events percona-xtradb-cluster-operator-6f69b446fd-twpr8_4662fb62-d87b-4d9b-b22e-563bbfb10138 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"bf50fc2a-f645-41ac-bd7b-79d18093eb21","apiVersion":"coordination.k8s.io/v1","resourceVersion":"63966"}, "reason": "LeaderElection"} 2024-07-17T11:47:33.897Z INFO successfully acquired lease pxc-operator/08db1feb.percona.com 2024-07-17T11:47:33.898Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2024-07-17T11:47:33.898Z INFO Starting Controller {"controller": "pxc-controller"} 2024-07-17T11:47:33.898Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2024-07-17T11:47:33.898Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2024-07-17T11:47:33.898Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"} 2024-07-17T11:47:33.898Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2024-07-17T11:47:34.007Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2024-07-17T11:47:34.007Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2024-07-17T11:47:34.015Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2024-07-17T11:49:56.881Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "547ee932-6cfb-4828-81ee-55032ab18bb9", "object": "smart-update-pxc"} 2024-07-17T11:49:56.936Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "547ee932-6cfb-4828-81ee-55032ab18bb9"} 2024-07-17T11:49:56.937Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "547ee932-6cfb-4828-81ee-55032ab18bb9", "object": "smart-update-pxc"} 2024-07-17T11:49:57.019Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "547ee932-6cfb-4828-81ee-55032ab18bb9", "object": "smart-update-pxc-unready"} 2024-07-17T11:49:57.133Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "547ee932-6cfb-4828-81ee-55032ab18bb9", "object": "smart-update-proxysql"} 2024-07-17T11:49:57.251Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "547ee932-6cfb-4828-81ee-55032ab18bb9", "object": "smart-update-proxysql"} 2024-07-17T11:49:57.393Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "547ee932-6cfb-4828-81ee-55032ab18bb9", "object": "smart-update-proxysql-unready"} 2024-07-17T11:49:57.689Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "4ed4e586-c25e-4da9-958b-c4b978a5c1e1", "object": "smart-update-pxc"} 2024-07-17T11:49:57.720Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "4ed4e586-c25e-4da9-958b-c4b978a5c1e1", "object": "smart-update-proxysql"} 2024-07-17T11:49:57.730Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "4ed4e586-c25e-4da9-958b-c4b978a5c1e1"} 2024-07-17T11:50:02.599Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "cc7a564c-4e33-4992-a1fa-8943fae86c45"} 2024-07-17T11:50:07.669Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "45c20c2d-9061-4c58-98c3-130f22691f5b"} 2024-07-17T11:50:12.740Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "3107602d-1ef0-4eda-9b73-4d85f746393e"} 2024-07-17T11:50:12.838Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "faf007de-ec1e-4e26-8e6f-fe7ca70522d7"} 2024-07-17T11:50:17.879Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0a18d826-8859-4d65-9729-a3cc579efeee"} 2024-07-17T11:50:23.229Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "85373f0b-0a20-4f65-91d5-fee09c4a1c16"} 2024-07-17T11:50:28.528Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "f02953ea-7301-4ba9-a1c1-8a191fbf9a76"} 2024-07-17T11:50:33.588Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "204070b2-ce35-4973-85f7-4815222aa9b0"} 2024-07-17T11:51:16.310Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "7dbec754-01a5-4545-881a-4d71216ce81c", "user": "operator"} 2024-07-17T11:51:16.351Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "7dbec754-01a5-4545-881a-4d71216ce81c", "user": "monitor"} 2024-07-17T11:51:16.423Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "7dbec754-01a5-4545-881a-4d71216ce81c"} 2024-07-17T11:51:16.463Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "7dbec754-01a5-4545-881a-4d71216ce81c"} 2024-07-17T11:51:16.499Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "7dbec754-01a5-4545-881a-4d71216ce81c", "user": "xtrabackup"} 2024-07-17T11:51:16.558Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "7dbec754-01a5-4545-881a-4d71216ce81c"} 2024-07-17T11:51:16.601Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "7dbec754-01a5-4545-881a-4d71216ce81c", "user": "replication"} 2024-07-17T11:51:17.636Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "7dbec754-01a5-4545-881a-4d71216ce81c", "err": "get primary pxc pod: not found"} 2024-07-17T11:51:22.321Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "26cd7059-f74b-4473-84e1-80db1ec7ebc1", "err": "get primary pxc pod: not found"} 2024-07-17T11:51:28.467Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "d4d81863-42a0-4ceb-93ea-d39bda403619", "err": "get primary pxc pod: not found"} 2024-07-17T11:53:45.726Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "f6660094-f95b-4b12-b226-fb8e7116f4ce", "user": "root"} 2024-07-17T11:53:46.879Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "f6660094-f95b-4b12-b226-fb8e7116f4ce", "new version": "8.0.19-10"} 2024-07-17T11:53:48.301Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "f6660094-f95b-4b12-b226-fb8e7116f4ce"} 2024-07-17T11:53:53.237Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "3e827c9d-766c-487a-aca8-7af990f9e1d2"} 2024-07-17T11:53:59.480Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "51542d47-5499-4554-ae67-4a37d8af61ac"} 2024-07-17T11:54:05.718Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "8e248518-96c4-4545-918e-6a9089d2efb9"} 2024-07-17T11:54:11.961Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "9fb5743e-4223-48b6-9c32-ee56ea27127b"} 2024-07-17T11:54:18.227Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "fa14c81e-d678-4d80-9d91-086e04af051d"} 2024-07-17T11:54:24.897Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "8025bddf-ba34-40c9-876f-7fb81d92d73c"} 2024-07-17T11:54:31.356Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "4703b691-2eba-42fe-b378-1ace8ab3e327"} 2024-07-17T11:54:37.589Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "da21aaf2-bc9c-445b-9712-7d66b9c6b42c"} 2024-07-17T11:54:43.839Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "91d88f27-eb02-4f94-9f2b-013733fc85ac"} 2024-07-17T11:54:50.153Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6f43f0b6-7654-4d9d-94cd-1ddf1c8b4de2"} 2024-07-17T11:54:50.656Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "object": "smart-update-pxc"} 2024-07-17T11:54:51.749Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5"} 2024-07-17T11:54:51.756Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-2"} 2024-07-17T11:54:51.756Z INFO primary pod {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-0.smart-update-pxc.smart-update1-3747.svc.cluster.local"} 2024-07-17T11:54:53.523Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5"} 2024-07-17T11:55:11.978Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-2", "reason": "PodInitializing"} 2024-07-17T11:56:22.016Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-2"} 2024-07-17T11:57:12.042Z INFO pod present in hostgroups {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-2"} 2024-07-17T11:57:12.049Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-1"} 2024-07-17T11:57:12.049Z INFO pod is online {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-2"} 2024-07-17T11:57:32.257Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-1", "reason": "PodInitializing"} 2024-07-17T11:57:42.257Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-1", "reason": "PodInitializing"} 2024-07-17T11:58:52.303Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-1"} 2024-07-17T12:00:02.325Z INFO pod present in hostgroups {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-1"} 2024-07-17T12:00:02.333Z INFO apply changes to primary pod {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-0"} 2024-07-17T12:00:02.333Z INFO pod is online {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-1"} 2024-07-17T12:00:22.524Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-0", "reason": "PodInitializing"} 2024-07-17T12:01:22.602Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-0"} 2024-07-17T12:02:12.629Z INFO pod present in hostgroups {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-0"} 2024-07-17T12:02:12.636Z INFO pod is online {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "pod name": "smart-update-pxc-0"} 2024-07-17T12:02:12.636Z INFO smart update finished {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5"} 2024-07-17T12:02:12.715Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0756f6e4-0d76-4aef-8c1d-814cfd0930a5", "new version": "8.0.36-28.1"} 2024-07-17T12:02:15.675Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "284479b1-f851-481d-9823-5b9f688b7b7d"} 2024-07-17T12:02:20.456Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "a55fbceb-8fee-4ce3-bfee-f907f1163ca4"} 2024-07-17T12:02:26.708Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "37c5d363-ca1b-4a8e-b7f1-0b46af0bb98e"} 2024-07-17T12:02:32.976Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "527d5c34-b2f8-4987-bb57-3f3806d9af53"} 2024-07-17T12:02:39.776Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "d2446bab-4a49-48b7-a869-f86e473981c6"} 2024-07-17T12:02:46.158Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "9cc47c45-fbd4-42a6-b0c5-41085deeb41d"} 2024-07-17T12:03:23.824Z ERROR Update status {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "99df0ce4-2b26-4672-83f7-36d10d7c3243", "error": "write status: PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found", "errorVerbose": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found\nwrite status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:157\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:140\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:203\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-17T12:03:41.582Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "bfdf9822-70b2-4990-8c1a-9400701ac425", "object": "smart-update-pxc"} 2024-07-17T12:03:41.653Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "bfdf9822-70b2-4990-8c1a-9400701ac425", "object": "smart-update-haproxy"} 2024-07-17T12:03:41.682Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "bfdf9822-70b2-4990-8c1a-9400701ac425", "object": "smart-update-pxc"} 2024-07-17T12:03:41.682Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "bfdf9822-70b2-4990-8c1a-9400701ac425"} 2024-07-17T12:03:41.761Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "bfdf9822-70b2-4990-8c1a-9400701ac425", "object": "smart-update-pxc-unready"} 2024-07-17T12:03:41.823Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "bfdf9822-70b2-4990-8c1a-9400701ac425", "object": "smart-update-haproxy"} 2024-07-17T12:03:41.886Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "bfdf9822-70b2-4990-8c1a-9400701ac425", "object": "smart-update-haproxy"} 2024-07-17T12:03:42.165Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "bfdf9822-70b2-4990-8c1a-9400701ac425", "object": "smart-update-haproxy-replicas"} 2024-07-17T12:03:42.316Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 98121993-19b0-41ef-82c3-3fe4c123fb72 2024-07-17T12:03:42.472Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "31b8b3d9-2d81-4f62-9a40-a98d220e4071", "object": "smart-update-pxc"} 2024-07-17T12:03:42.548Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "31b8b3d9-2d81-4f62-9a40-a98d220e4071"} 2024-07-17T12:03:43.853Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "edf53b15-2105-4eb5-9ed3-0a2878949389"} 2024-07-17T12:03:48.931Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "30c91618-b20e-4fa0-9a30-e47fc1ce3e46"} 2024-07-17T12:03:54.013Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "149bf136-7b18-40d1-a334-79ef713cd795"} 2024-07-17T12:03:59.079Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "79fa0578-146e-4ec0-8449-fab487717220"} 2024-07-17T12:04:04.206Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "8d155fc7-751a-4aea-9612-522fb5a83bf9"} 2024-07-17T12:04:09.284Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "ae7d2404-5c97-4cbf-bca3-95fbb59ffda0"} 2024-07-17T12:04:14.363Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "224c2f57-8c76-4852-8034-2c88df1f3b42"} 2024-07-17T12:04:19.426Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "56fbf52c-f143-403e-949b-77a1d9941076"} 2024-07-17T12:04:24.488Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "64e119a6-1388-4229-a567-3a2594a3b0e6"} 2024-07-17T12:04:29.551Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0885e50e-f9ab-451c-af3f-481e8c04ccc4"} 2024-07-17T12:04:34.613Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "94b71cca-099b-404d-ba32-525822cb1e72"} 2024-07-17T12:04:39.676Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "32b95059-6b51-45c6-a825-5547153cc32b"} 2024-07-17T12:04:44.765Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "fcc59cc1-0f5a-435e-bdf5-e7d83b5e4c89"} 2024-07-17T12:04:49.847Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "cd7cc816-297f-4028-b925-37a1ae2274b5"} 2024-07-17T12:04:54.911Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "184867a3-4839-40ed-b4f7-7b9c6613cf3d"} 2024-07-17T12:05:00.006Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "06432e91-fb6d-4e14-b920-d55d487c6bcc"} 2024-07-17T12:05:00.201Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "8b62c7dc-8f30-4097-8c14-b1803ed4f78e"} 2024-07-17T12:05:05.194Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "a99ffa0b-7f5a-4b16-83e4-41c29888adde"} 2024-07-17T12:05:10.355Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "a06a7eff-8097-42d5-86a2-b52ee4732dab"} 2024-07-17T12:05:15.507Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "1178a8c6-ba5a-4a9a-8757-abe043e63f2d"} 2024-07-17T12:05:20.650Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "948fb314-e5de-4844-a7c8-20bc44176bec"} 2024-07-17T12:07:26.185Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "583066a9-bc1f-46d1-af55-9611f93f3a48", "new version": "8.0.19-10"} 2024-07-17T12:08:31.272Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6b1bd07e-5835-4476-96cb-ca54f313c634", "object": "smart-update-pxc"} 2024-07-17T12:08:32.332Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6b1bd07e-5835-4476-96cb-ca54f313c634"} 2024-07-17T12:08:32.347Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6b1bd07e-5835-4476-96cb-ca54f313c634", "pod name": "smart-update-pxc-2"} 2024-07-17T12:08:32.347Z INFO primary pod {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6b1bd07e-5835-4476-96cb-ca54f313c634", "pod name": "smart-update-pxc-0.smart-update-pxc.smart-update1-3747"} 2024-07-17T12:09:52.575Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6b1bd07e-5835-4476-96cb-ca54f313c634", "pod name": "smart-update-pxc-2"} 2024-07-17T12:09:52.599Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6b1bd07e-5835-4476-96cb-ca54f313c634", "pod name": "smart-update-pxc-1"} 2024-07-17T12:10:13.470Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6b1bd07e-5835-4476-96cb-ca54f313c634", "pod name": "smart-update-pxc-1", "reason": "PodInitializing"} 2024-07-17T12:11:23.649Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6b1bd07e-5835-4476-96cb-ca54f313c634", "pod name": "smart-update-pxc-1"} 2024-07-17T12:11:23.667Z INFO apply changes to primary pod {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6b1bd07e-5835-4476-96cb-ca54f313c634", "pod name": "smart-update-pxc-0"} 2024-07-17T12:11:43.913Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6b1bd07e-5835-4476-96cb-ca54f313c634", "pod name": "smart-update-pxc-0", "reason": "PodInitializing"} 2024-07-17T12:12:53.946Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6b1bd07e-5835-4476-96cb-ca54f313c634", "pod name": "smart-update-pxc-0"} 2024-07-17T12:12:53.969Z INFO smart update finished {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6b1bd07e-5835-4476-96cb-ca54f313c634"} 2024-07-17T12:12:54.052Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6b1bd07e-5835-4476-96cb-ca54f313c634", "new version": "8.0.36-28.1"} 2024-07-17T12:13:19.796Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "fe6d9918-a4ed-4561-850b-1e09cef36ee8"} 2024-07-17T12:13:53.940Z ERROR Update status {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "beee6954-ef48-4fc7-ac15-f680683f65bb", "error": "write status: PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found", "errorVerbose": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found\nwrite status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:157\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:140\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:203\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-17T12:14:12.706Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0f008e84-d8db-442d-ae0c-98224b8e8200", "object": "smart-update-pxc"} 2024-07-17T12:14:12.756Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0f008e84-d8db-442d-ae0c-98224b8e8200", "object": "smart-update-haproxy"} 2024-07-17T12:14:12.797Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0f008e84-d8db-442d-ae0c-98224b8e8200", "object": "smart-update-pxc"} 2024-07-17T12:14:12.797Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0f008e84-d8db-442d-ae0c-98224b8e8200"} 2024-07-17T12:14:12.872Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0f008e84-d8db-442d-ae0c-98224b8e8200", "object": "smart-update-pxc-unready"} 2024-07-17T12:14:12.930Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0f008e84-d8db-442d-ae0c-98224b8e8200", "object": "smart-update-haproxy"} 2024-07-17T12:14:13.052Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0f008e84-d8db-442d-ae0c-98224b8e8200", "object": "smart-update-haproxy"} 2024-07-17T12:14:13.190Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0f008e84-d8db-442d-ae0c-98224b8e8200", "object": "smart-update-haproxy"} 2024-07-17T12:14:13.252Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0f008e84-d8db-442d-ae0c-98224b8e8200", "object": "smart-update-haproxy"} 2024-07-17T12:14:13.328Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "0f008e84-d8db-442d-ae0c-98224b8e8200", "object": "smart-update-haproxy-replicas"} 2024-07-17T12:14:13.402Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 963b8e29-a7db-4a18-babb-f1bae4b14c42 2024-07-17T12:14:13.478Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6710af44-fcee-4644-8f96-325011c8739a", "object": "smart-update-pxc"} 2024-07-17T12:14:13.511Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6710af44-fcee-4644-8f96-325011c8739a"} 2024-07-17T12:14:13.970Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "7aa488d3-6560-4a3c-975f-54426fb1f5c5"} 2024-07-17T12:14:19.031Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "68219c0c-18b7-4a5a-ba3d-13e2445d86e5"} 2024-07-17T12:14:24.095Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "5ac70be6-0c46-4e60-9cfb-8ed91ae2c573"} 2024-07-17T12:14:29.164Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "d7d1d83d-deac-473b-92ac-ed270af5d785"} 2024-07-17T12:14:34.227Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "8851622d-b3da-4ede-b694-d01744754ab6"} 2024-07-17T12:14:39.299Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "98116948-45d5-4dab-bfaf-c9524138ce7d"} 2024-07-17T12:14:44.366Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "3cbea47f-e7da-4751-bc6d-c1b9d39a9756"} 2024-07-17T12:14:49.436Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "f5392d3a-c746-4e7f-a686-5314374ce331"} 2024-07-17T12:14:54.507Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "ae63e309-858d-4f5b-ba95-88acae5096e1"} 2024-07-17T12:14:59.577Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "feb64229-adba-462a-b839-c68d39defa13"} 2024-07-17T12:15:04.641Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "905f3b12-6b35-481c-8295-64f553ddcf5c"} 2024-07-17T12:15:09.710Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "ce71a58f-0317-4432-a894-eea66d4e64e6"} 2024-07-17T12:15:14.801Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "b706fe8c-4a3a-4c64-8bb2-02e0e7acfb70"} 2024-07-17T12:15:19.949Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "d0caccf3-ba96-49f1-a360-94ec12a194fa"} 2024-07-17T12:15:25.037Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "5b10a55b-8538-4c4a-a3cc-d3924f5e0c26"} 2024-07-17T12:15:30.120Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "5f86835a-a84d-45a0-b7fa-8c721982dc1b"} 2024-07-17T12:15:30.264Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "e26abbd3-7c09-4af2-837c-ad3c2755b694"} 2024-07-17T12:15:35.273Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "c28ec2f1-f6bf-4fc7-b495-0e5a5c44d233"} 2024-07-17T12:15:40.455Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "7d451b19-4dc8-4fb1-80d5-22c7927be444"} 2024-07-17T12:15:45.624Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "d6f0011c-ea6e-4b20-8aa8-30377870ff0b"} 2024-07-17T12:15:50.793Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "6c41d354-86eb-40f2-8e5d-c88910bfe382"} 2024-07-17T12:17:56.059Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "abef185e-8771-4191-bf66-1c1eb2e5e5fb", "new version": "8.0.19-10"} 2024-07-17T12:19:03.801Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26", "object": "smart-update-pxc"} 2024-07-17T12:19:04.865Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26"} 2024-07-17T12:19:04.878Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26", "pod name": "smart-update-pxc-2"} 2024-07-17T12:19:04.878Z INFO primary pod {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26", "pod name": "smart-update-pxc-0.smart-update-pxc.smart-update1-3747"} 2024-07-17T12:19:25.106Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26", "pod name": "smart-update-pxc-2", "reason": "PodInitializing"} 2024-07-17T12:20:35.142Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26", "pod name": "smart-update-pxc-2"} 2024-07-17T12:20:35.169Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26", "pod name": "smart-update-pxc-1"} 2024-07-17T12:20:55.390Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26", "pod name": "smart-update-pxc-1", "reason": "PodInitializing"} 2024-07-17T12:21:05.390Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26", "pod name": "smart-update-pxc-1", "reason": "PodInitializing"} 2024-07-17T12:22:15.419Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26", "pod name": "smart-update-pxc-1"} 2024-07-17T12:22:15.438Z INFO apply changes to primary pod {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26", "pod name": "smart-update-pxc-0"} 2024-07-17T12:22:35.654Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26", "pod name": "smart-update-pxc-0", "reason": "PodInitializing"} 2024-07-17T12:23:35.689Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26", "pod name": "smart-update-pxc-0"} 2024-07-17T12:23:35.713Z INFO smart update finished {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26"} 2024-07-17T12:23:35.800Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "2ce9f813-2346-4996-bc96-215690ee9d26", "new version": "8.0.36-28.1"} 2024-07-17T12:24:28.685Z ERROR Update status {"controller": "pxc-controller", "namespace": "smart-update1-3747", "name": "smart-update", "reconcileID": "b3a648bc-0708-4afb-a80a-8af77aa8e16c", "error": "write status: PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found", "errorVerbose": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found\nwrite status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:157\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:140\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:203\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:205 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:265 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.d3EoAZ0Zel ++ mktemp + local LAST_ERR=/tmp/tmp.2iL8EcmTz3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.d3EoAZ0Zel No resources found + cat /tmp/tmp.2iL8EcmTz3 + rm /tmp/tmp.d3EoAZ0Zel /tmp/tmp.2iL8EcmTz3 + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.Y7cd2wBUPe ++ mktemp + local LAST_ERR=/tmp/tmp.p7vYTLyZQR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Y7cd2wBUPe No resources found + cat /tmp/tmp.p7vYTLyZQR + rm /tmp/tmp.Y7cd2wBUPe /tmp/tmp.p7vYTLyZQR + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.Yej9jjadeu ++ mktemp + local LAST_ERR=/tmp/tmp.ZF2Rd2OpVl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Yej9jjadeu No resources found + cat /tmp/tmp.ZF2Rd2OpVl + rm /tmp/tmp.Yej9jjadeu /tmp/tmp.ZF2Rd2OpVl + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.J4rsFFCQ8Y ++ mktemp + local LAST_ERR=/tmp/tmp.um1NUDdofX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.J4rsFFCQ8Y validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.um1NUDdofX + rm /tmp/tmp.J4rsFFCQ8Y /tmp/tmp.um1NUDdofX + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace smart-update1-3747 + rm -rf /tmp/tmp.eDPRkBgipg + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ mktemp + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.zsve10Pzep test passed ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.lWtj6yE4ew ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.AiW0WSZJyh + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.HKMuCLvLEA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace smart-update1-3747 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator