Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/logs/smart-update1-8-0.log WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 + API=pxc.percona.com/v9-9-9 + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + CLUSTER=smart-update + CLUSTER_SIZE=3 + PROXY_SIZE=2 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + PXC_VER=8.0 + TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + VS_URL=http://version-service + VS_PORT=11000 + VS_ENDPOINT=http://version-service:11000 + main + create_infra smart-update1-3222 + local ns=smart-update1-3222 + '[' -n pxc-operator ']' + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl get pxc --all-namespaces -o wide + kubectl patch pxc -n smart-update1-20545 smart-update --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/smart-update patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.oRGFGQ6Yl8 ++ mktemp + local LAST_ERR=/tmp/tmp.Xi92Chrqkm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oRGFGQ6Yl8 perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.Xi92Chrqkm + rm /tmp/tmp.oRGFGQ6Yl8 /tmp/tmp.Xi92Chrqkm + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.rGoAOjk7ie ++ mktemp + local LAST_ERR=/tmp/tmp.jDxky7S54q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rGoAOjk7ie No resources found + cat /tmp/tmp.jDxky7S54q + rm /tmp/tmp.rGoAOjk7ie /tmp/tmp.jDxky7S54q + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.Oc8tJAyUrl ++ mktemp + local LAST_ERR=/tmp/tmp.Szot2rtE1z + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Oc8tJAyUrl No resources found + cat /tmp/tmp.Szot2rtE1z + rm /tmp/tmp.Oc8tJAyUrl /tmp/tmp.Szot2rtE1z + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + xargs kubectl delete ns + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.XasdZ5tFNQ ++ mktemp + local LAST_OUT=/tmp/tmp.0aKAKUSP1i ++ mktemp + local LAST_ERR=/tmp/tmp.IVaRVlJOvy + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.Bj8FpXnHdJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + for i in '$(seq 0 2)' + set +e + kubectl get ns + awk '{print$1}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0aKAKUSP1i + cat /tmp/tmp.Bj8FpXnHdJ + rm /tmp/tmp.0aKAKUSP1i /tmp/tmp.Bj8FpXnHdJ + return 0 namespace "cert-manager" deleted namespace "smart-update1-20545" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XasdZ5tFNQ namespace "pxc-operator" deleted + cat /tmp/tmp.IVaRVlJOvy + rm /tmp/tmp.XasdZ5tFNQ /tmp/tmp.IVaRVlJOvy + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'namespace/pxc-operator - ' namespace/pxc-operator - + set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.XqYCYXgtb1 ++ mktemp + local LAST_ERR=/tmp/tmp.B8JMbX7mNj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XqYCYXgtb1 namespace/pxc-operator created + cat /tmp/tmp.B8JMbX7mNj + rm /tmp/tmp.XqYCYXgtb1 /tmp/tmp.B8JMbX7mNj + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.dwR7NUAwhn +++ mktemp ++ local LAST_ERR=/tmp/tmp.PsmEc6BoFI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dwR7NUAwhn ++ cat /tmp/tmp.PsmEc6BoFI ++ rm /tmp/tmp.dwR7NUAwhn /tmp/tmp.PsmEc6BoFI ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1750-de45ff75-2-cluster5 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.5BKyFnsT6w ++ mktemp + local LAST_ERR=/tmp/tmp.m86BQLsBcB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1750-de45ff75-2-cluster5 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5BKyFnsT6w Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1750-de45ff75-2-cluster5" modified. + cat /tmp/tmp.m86BQLsBcB + rm /tmp/tmp.5BKyFnsT6w /tmp/tmp.m86BQLsBcB + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.s2ilmiwH9z ++ mktemp + local LAST_ERR=/tmp/tmp.YFfANNLS0L + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.s2ilmiwH9z customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.YFfANNLS0L + rm /tmp/tmp.s2ilmiwH9z /tmp/tmp.YFfANNLS0L + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/deploy/cw-rbac.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.wfENNKvNCu ++ mktemp + local LAST_ERR=/tmp/tmp.5sUAWizjYO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wfENNKvNCu clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.5sUAWizjYO + rm /tmp/tmp.wfENNKvNCu /tmp/tmp.5sUAWizjYO + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1750-de45ff75^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.sWspCyuNb6 ++ mktemp + local LAST_ERR=/tmp/tmp.42vQW4OR7W + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sWspCyuNb6 deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.42vQW4OR7W + rm /tmp/tmp.sWspCyuNb6 /tmp/tmp.42vQW4OR7W + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.P2AsFITMza ++ mktemp + local LAST_ERR=/tmp/tmp.m6EL1I8Vsf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.P2AsFITMza pod/percona-xtradb-cluster-operator-7c5cdd5dd9-jpd4z condition met + cat /tmp/tmp.m6EL1I8Vsf + rm /tmp/tmp.P2AsFITMza /tmp/tmp.m6EL1I8Vsf + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.RuywkyYXic +++ mktemp ++ local LAST_ERR=/tmp/tmp.j0W8H779Fg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RuywkyYXic ++ cat /tmp/tmp.j0W8H779Fg ++ rm /tmp/tmp.RuywkyYXic /tmp/tmp.j0W8H779Fg ++ return 0 + wait_pod percona-xtradb-cluster-operator-7c5cdd5dd9-jpd4z 480 pxc-operator + local pod=percona-xtradb-cluster-operator-7c5cdd5dd9-jpd4z + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-7c5cdd5dd9-jpd4z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-7c5cdd5dd9-jpd4z condition met percona-xtradb-cluster-operator-7c5cdd5dd9-jpd4z.Ok + sleep 3 + create_namespace smart-update1-3222 + local namespace=smart-update1-3222 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl api-resources ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces smart-update1-3222' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces smart-update1-3222 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace smart-update1-3222 + xargs kubectl delete ns ++ mktemp ++ mktemp + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + local LAST_OUT=/tmp/tmp.v3XMsYpu39 ++ mktemp + local LAST_OUT=/tmp/tmp.Npv2s36byt ++ mktemp + local LAST_ERR=/tmp/tmp.4wtJLnIiUQ + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.Lin8a6XoIz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-3222 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Npv2s36byt + cat /tmp/tmp.Lin8a6XoIz + rm /tmp/tmp.Npv2s36byt /tmp/tmp.Lin8a6XoIz + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-3222 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update1-3222 Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.v3XMsYpu39 + cat /tmp/tmp.4wtJLnIiUQ Error from server (NotFound): namespaces "smart-update1-3222" not found + rm /tmp/tmp.v3XMsYpu39 /tmp/tmp.4wtJLnIiUQ + return 1 + : + wait_for_delete namespace/smart-update1-3222 + local res=namespace/smart-update1-3222 + echo -n 'namespace/smart-update1-3222 - ' namespace/smart-update1-3222 - + set +o xtrace Error from server (NotFound): namespaces "smart-update1-3222" not found + desc 'create namespace smart-update1-3222' + set +o xtrace ----------------------------------------------------------------------------------- create namespace smart-update1-3222 ----------------------------------------------------------------------------------- + kubectl_bin create namespace smart-update1-3222 ++ mktemp + local LAST_OUT=/tmp/tmp.R3g1dkuts4 ++ mktemp + local LAST_ERR=/tmp/tmp.T3OXQlXSD6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace smart-update1-3222 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.R3g1dkuts4 namespace/smart-update1-3222 created + cat /tmp/tmp.T3OXQlXSD6 + rm /tmp/tmp.R3g1dkuts4 /tmp/tmp.T3OXQlXSD6 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.bcmMXoFYYx +++ mktemp ++ local LAST_ERR=/tmp/tmp.EUwDooy5dx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bcmMXoFYYx ++ cat /tmp/tmp.EUwDooy5dx ++ rm /tmp/tmp.bcmMXoFYYx /tmp/tmp.EUwDooy5dx ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1750-de45ff75-2-cluster5 --namespace=smart-update1-3222 ++ mktemp + local LAST_OUT=/tmp/tmp.IgYtgGvlX3 ++ mktemp + local LAST_ERR=/tmp/tmp.tybWBaaOP2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1750-de45ff75-2-cluster5 --namespace=smart-update1-3222 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IgYtgGvlX3 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1750-de45ff75-2-cluster5" modified. + cat /tmp/tmp.tybWBaaOP2 + rm /tmp/tmp.IgYtgGvlX3 /tmp/tmp.tybWBaaOP2 + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.EKqK8HYTBZ ++ mktemp + local LAST_ERR=/tmp/tmp.SU9QSO2Dip + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EKqK8HYTBZ secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.SU9QSO2Dip + rm /tmp/tmp.EKqK8HYTBZ /tmp/tmp.SU9QSO2Dip + return 0 + deploy_version_service + desc 'install version service' + set +o xtrace ----------------------------------------------------------------------------------- install version service ----------------------------------------------------------------------------------- + kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json ++ mktemp + local LAST_OUT=/tmp/tmp.cKLNeHcal1 ++ mktemp + local LAST_ERR=/tmp/tmp.jWsbzD5Zz1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cKLNeHcal1 configmap/versions created + cat /tmp/tmp.jWsbzD5Zz1 + rm /tmp/tmp.cKLNeHcal1 /tmp/tmp.jWsbzD5Zz1 + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.fUHf1RVxZ1 ++ mktemp + local LAST_ERR=/tmp/tmp.qNZyAo3jTO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fUHf1RVxZ1 deployment.apps/version-service created service/version-service created + cat /tmp/tmp.qNZyAo3jTO + rm /tmp/tmp.fUHf1RVxZ1 /tmp/tmp.qNZyAo3jTO + return 0 + sleep 10 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.AunH9RzDmh ++ mktemp + local LAST_ERR=/tmp/tmp.tK4G234u8P + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AunH9RzDmh namespace/cert-manager created + cat /tmp/tmp.tK4G234u8P + rm /tmp/tmp.AunH9RzDmh /tmp/tmp.tK4G234u8P + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.WUXnYRtrX2 ++ mktemp + local LAST_ERR=/tmp/tmp.80mU2dwHpc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WUXnYRtrX2 namespace/cert-manager labeled + cat /tmp/tmp.80mU2dwHpc + rm /tmp/tmp.WUXnYRtrX2 /tmp/tmp.80mU2dwHpc + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.tBWTLFHRk9 ++ mktemp + local LAST_ERR=/tmp/tmp.sMYdIMwDzX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tBWTLFHRk9 namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews configured role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection configured rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.sMYdIMwDzX Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.tBWTLFHRk9 /tmp/tmp.sMYdIMwDzX + return 0 + '[' '' == 4.10 ']' + sleep 70 ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ grep :8.0 +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++ head -n1 ++ tail -n3 ++++ grep -c percona-xtradb-cluster-operator ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ sort -V +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hwakGIfsp1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UbZGz7EVuf +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.hwakGIfsp1 +++ cat /tmp/tmp.UbZGz7EVuf +++ rm /tmp/tmp.hwakGIfsp1 /tmp/tmp.UbZGz7EVuf +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-7c5cdd5dd9-jpd4z -n pxc-operator -- curl -s http://version-service.smart-update1-3222.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.fi5LAz2aia +++ mktemp ++ local LAST_ERR=/tmp/tmp.tf0Fm5VKO3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-7c5cdd5dd9-jpd4z -n pxc-operator -- curl -s http://version-service.smart-update1-3222.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fi5LAz2aia ++ cat /tmp/tmp.tf0Fm5VKO3 Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.fi5LAz2aia /tmp/tmp.tf0Fm5VKO3 ++ return 0 + IMAGE_PXC=percona/percona-xtradb-cluster:8.0.19-10.1 + desc 'patch crd' + set +o xtrace ----------------------------------------------------------------------------------- patch crd ----------------------------------------------------------------------------------- + kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' ++ mktemp + local LAST_OUT=/tmp/tmp.Q3tdc8Q4pi ++ mktemp + local LAST_ERR=/tmp/tmp.WrvR3qYuJ1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Q3tdc8Q4pi customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched + cat /tmp/tmp.WrvR3qYuJ1 + rm /tmp/tmp.Q3tdc8Q4pi /tmp/tmp.WrvR3qYuJ1 + return 0 + desc 'Updating ProxySQL PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- Updating ProxySQL PXC cluster ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/conf/smart-update.yml /tmp/tmp.g7q1LAQh2L/smart-update.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1750-de45ff75"' /tmp/tmp.g7q1LAQh2L/smart-update.yml + spinup_pxc smart-update /tmp/tmp.g7q1LAQh2L/smart-update.yml + local cluster=smart-update + local config=/tmp/tmp.g7q1LAQh2L/smart-update.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.zCJExPOrNh ++ mktemp + local LAST_ERR=/tmp/tmp.Aywfkjuoqu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zCJExPOrNh secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.Aywfkjuoqu + rm /tmp/tmp.zCJExPOrNh /tmp/tmp.Aywfkjuoqu + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/client.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1750-de45ff75#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-3222~ + local LAST_OUT=/tmp/tmp.P1Z5Ope1gY + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' ++ mktemp + local LAST_ERR=/tmp/tmp.DmZryB9o6b + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.P1Z5Ope1gY deployment.apps/pxc-client created + cat /tmp/tmp.DmZryB9o6b + rm /tmp/tmp.P1Z5Ope1gY /tmp/tmp.DmZryB9o6b + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.g7q1LAQh2L/smart-update.yml + '[' -z '' ']' + cat_config /tmp/tmp.g7q1LAQh2L/smart-update.yml + kubectl_bin apply -f - + cat /tmp/tmp.g7q1LAQh2L/smart-update.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + local LAST_OUT=/tmp/tmp.4LjHHE8GCX + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1750-de45ff75#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_ERR=/tmp/tmp.kWvncWOib3 + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-3222~ ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4LjHHE8GCX perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.kWvncWOib3 + rm /tmp/tmp.4LjHHE8GCX /tmp/tmp.kWvncWOib3 + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lj3qRzTyqt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.aSSj7orc3z +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.lj3qRzTyqt +++ cat /tmp/tmp.aSSj7orc3z +++ rm /tmp/tmp.lj3qRzTyqt /tmp/tmp.aSSj7orc3z +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Rtcfa3QGGl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zyROPQ8Vl8 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Rtcfa3QGGl +++ cat /tmp/tmp.zyROPQ8Vl8 +++ rm /tmp/tmp.Rtcfa3QGGl /tmp/tmp.zyROPQ8Vl8 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-proxysql ++ return + local proxy=smart-update-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3222 ++ mktemp + local LAST_OUT=/tmp/tmp.2I9wdgyer6 ++ mktemp + local LAST_ERR=/tmp/tmp.xHGdNTFW85 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3222 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3222 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3222 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.2I9wdgyer6 + cat /tmp/tmp.xHGdNTFW85 error: no matching resources found + rm /tmp/tmp.2I9wdgyer6 /tmp/tmp.xHGdNTFW85 + return 1 + true + wait_for_running smart-update-proxysql 1 + local name=smart-update-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-proxysql-0 480 + local pod=smart-update-proxysql-0 + local max_retry=480 + local ns= ++ echo smart-update-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace Error from server (NotFound): pods "smart-update-proxysql-0" not found smart-update-proxysql-0.............Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met smart-update-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met smart-update-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-proxysql -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eNWlvhbQio +++ mktemp ++ local LAST_ERR=/tmp/tmp.NDRRyEAZgK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eNWlvhbQio ++ cat /tmp/tmp.NDRRyEAZgK ++ rm /tmp/tmp.eNWlvhbQio /tmp/tmp.NDRRyEAZgK ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-proxysql -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ztEgOPTuW2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.w9aifM1cj3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ztEgOPTuW2 ++ cat /tmp/tmp.w9aifM1cj3 ++ rm /tmp/tmp.ztEgOPTuW2 /tmp/tmp.w9aifM1cj3 ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZnYHeXkZ7u +++ mktemp ++ local LAST_ERR=/tmp/tmp.55jeKwezqe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZnYHeXkZ7u ++ cat /tmp/tmp.55jeKwezqe ++ rm /tmp/tmp.ZnYHeXkZ7u /tmp/tmp.55jeKwezqe ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CQuUMvdZs2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.EWAa4ktW87 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CQuUMvdZs2 ++ cat /tmp/tmp.EWAa4ktW87 ++ rm /tmp/tmp.CQuUMvdZs2 /tmp/tmp.EWAa4ktW87 ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UZ1o10YB2n +++ mktemp ++ local LAST_ERR=/tmp/tmp.qO7Rxu1ByJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UZ1o10YB2n ++ cat /tmp/tmp.qO7Rxu1ByJ ++ rm /tmp/tmp.UZ1o10YB2n /tmp/tmp.qO7Rxu1ByJ ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uUp73pNkAp +++ mktemp ++ local LAST_ERR=/tmp/tmp.O8Gxl6wRBI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uUp73pNkAp ++ cat /tmp/tmp.O8Gxl6wRBI Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.uUp73pNkAp /tmp/tmp.O8Gxl6wRBI ++ return 0 + '[' '' ']' +++ get_proxy smart-update +++ local target_cluster=smart-update ++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.vUDOs4L1bC +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.lNZKUn1VKH ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.vUDOs4L1bC ++++ cat /tmp/tmp.lNZKUn1VKH ++++ rm /tmp/tmp.vUDOs4L1bC /tmp/tmp.lNZKUn1VKH ++++ return 0 +++ [[ '' == \t\r\u\e ]] ++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.Bx8QO2Sldc +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.9KSeeoIMOf ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.Bx8QO2Sldc ++++ cat /tmp/tmp.9KSeeoIMOf ++++ rm /tmp/tmp.Bx8QO2Sldc /tmp/tmp.9KSeeoIMOf ++++ return 0 +++ [[ true == \t\r\u\e ]] +++ echo smart-update-proxysql +++ return ++ get_proxy_primary '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=smart-update-proxysql-0 +++ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 proxysql +++ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' +++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' +++ local pod=smart-update-proxysql-0 +++ local container_name=proxysql +++ set +o xtrace ++ local ip=smart-update-pxc-0.smart-update-pxc.smart-update1-3222.svc.cluster.local +++ echo smart-update-pxc-0.smart-update-pxc.smart-update1-3222.svc.cluster.local +++ wc -l ++ '[' 1 '!=' 1 ']' ++ echo smart-update-pxc-0.smart-update-pxc.smart-update1-3222.svc.cluster.local ++ cut -d. -f1 + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.GRjyDahUhG ++ mktemp + local LAST_ERR=/tmp/tmp.7KBdfEMX7M + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GRjyDahUhG perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.7KBdfEMX7M + rm /tmp/tmp.GRjyDahUhG /tmp/tmp.7KBdfEMX7M + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update........................................................................ smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DICiGkqksk +++ mktemp ++ local LAST_ERR=/tmp/tmp.vF3p1nlLvt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DICiGkqksk ++ cat /tmp/tmp.vF3p1nlLvt ++ rm /tmp/tmp.DICiGkqksk /tmp/tmp.vF3p1nlLvt ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oSAOkKUjSX +++ mktemp ++ local LAST_ERR=/tmp/tmp.FnFBn8JxOr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oSAOkKUjSX ++ cat /tmp/tmp.FnFBn8JxOr ++ rm /tmp/tmp.oSAOkKUjSX /tmp/tmp.FnFBn8JxOr ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8eNGIIaO3M +++ mktemp ++ local LAST_ERR=/tmp/tmp.cR8yGec3yK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8eNGIIaO3M ++ cat /tmp/tmp.cR8yGec3yK ++ rm /tmp/tmp.8eNGIIaO3M /tmp/tmp.cR8yGec3yK ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2ZOUIoopvQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.DqdpU2jC0C ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2ZOUIoopvQ ++ cat /tmp/tmp.DqdpU2jC0C ++ rm /tmp/tmp.2ZOUIoopvQ /tmp/tmp.DqdpU2jC0C ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TWR0YQ9cK6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.jX1HJ5rKhS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TWR0YQ9cK6 ++ cat /tmp/tmp.jX1HJ5rKhS ++ rm /tmp/tmp.TWR0YQ9cK6 /tmp/tmp.jX1HJ5rKhS ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LulmfN6MWZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.eJ5JKhe84p ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LulmfN6MWZ ++ cat /tmp/tmp.eJ5JKhe84p ++ rm /tmp/tmp.LulmfN6MWZ /tmp/tmp.eJ5JKhe84p ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.663iZCITXy +++ mktemp ++ local LAST_ERR=/tmp/tmp.2TmpIZ7CTQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.663iZCITXy ++ cat /tmp/tmp.2TmpIZ7CTQ ++ rm /tmp/tmp.663iZCITXy /tmp/tmp.2TmpIZ7CTQ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LY5vdnamVq +++ mktemp ++ local LAST_ERR=/tmp/tmp.AN8g4pW2DB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LY5vdnamVq ++ cat /tmp/tmp.AN8g4pW2DB ++ rm /tmp/tmp.LY5vdnamVq /tmp/tmp.AN8g4pW2DB ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xEFfok6adA +++ mktemp ++ local LAST_ERR=/tmp/tmp.DseagPHqi9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xEFfok6adA ++ cat /tmp/tmp.DseagPHqi9 ++ rm /tmp/tmp.xEFfok6adA /tmp/tmp.DseagPHqi9 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 8 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3QMmpKrJxr +++ mktemp ++ local LAST_ERR=/tmp/tmp.81rSBpQbMT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3QMmpKrJxr ++ cat /tmp/tmp.81rSBpQbMT ++ rm /tmp/tmp.3QMmpKrJxr /tmp/tmp.81rSBpQbMT ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 9 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CjwCs5VkZj +++ mktemp ++ local LAST_ERR=/tmp/tmp.bH29dBVPbI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CjwCs5VkZj ++ cat /tmp/tmp.bH29dBVPbI ++ rm /tmp/tmp.CjwCs5VkZj /tmp/tmp.bH29dBVPbI ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 10 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v1BGNMJj33 +++ mktemp ++ local LAST_ERR=/tmp/tmp.rUsVQm9RWP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.v1BGNMJj33 ++ cat /tmp/tmp.rUsVQm9RWP ++ rm /tmp/tmp.v1BGNMJj33 /tmp/tmp.rUsVQm9RWP ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 11 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UJCF2yD392 +++ mktemp ++ local LAST_ERR=/tmp/tmp.MyStVr80ca ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UJCF2yD392 ++ cat /tmp/tmp.MyStVr80ca ++ rm /tmp/tmp.UJCF2yD392 /tmp/tmp.MyStVr80ca ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 12 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1w8fh5y5L8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FleQkmtSpp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1w8fh5y5L8 ++ cat /tmp/tmp.FleQkmtSpp ++ rm /tmp/tmp.1w8fh5y5L8 /tmp/tmp.FleQkmtSpp ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 13 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Iau56akndt +++ mktemp ++ local LAST_ERR=/tmp/tmp.MaoX0eoqyh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Iau56akndt ++ cat /tmp/tmp.MaoX0eoqyh ++ rm /tmp/tmp.Iau56akndt /tmp/tmp.MaoX0eoqyh ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.t40bWWBYjH +++ mktemp ++ local LAST_ERR=/tmp/tmp.W0rfqLwGNN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.t40bWWBYjH ++ cat /tmp/tmp.W0rfqLwGNN ++ rm /tmp/tmp.t40bWWBYjH /tmp/tmp.W0rfqLwGNN ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.FZY99tk0ix ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.OoCR87CQYS +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.FZY99tk0ix +++++ cat /tmp/tmp.OoCR87CQYS +++++ rm /tmp/tmp.FZY99tk0ix /tmp/tmp.OoCR87CQYS +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.4LTDSmUeld ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.UcFM6HMzeA +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.4LTDSmUeld +++++ cat /tmp/tmp.UcFM6HMzeA +++++ rm /tmp/tmp.4LTDSmUeld /tmp/tmp.UcFM6HMzeA +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-proxysql ++++ return +++ local cluster_proxy=smart-update-proxysql +++ echo proxysql ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lP09yNt3NR +++ mktemp ++ local LAST_ERR=/tmp/tmp.H48fM3CGdI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lP09yNt3NR ++ cat /tmp/tmp.H48fM3CGdI ++ rm /tmp/tmp.lP09yNt3NR /tmp/tmp.H48fM3CGdI ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.91q5NJ7WXS +++ mktemp ++ local LAST_ERR=/tmp/tmp.a6PNqnT4q2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.91q5NJ7WXS ++ cat /tmp/tmp.a6PNqnT4q2 ++ rm /tmp/tmp.91q5NJ7WXS /tmp/tmp.a6PNqnT4q2 ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Gj21oEu6Cn +++ mktemp ++ local LAST_ERR=/tmp/tmp.O6kLEtw99d ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Gj21oEu6Cn ++ cat /tmp/tmp.O6kLEtw99d ++ rm /tmp/tmp.Gj21oEu6Cn /tmp/tmp.O6kLEtw99d ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ParhkHP82a +++ mktemp ++ local LAST_ERR=/tmp/tmp.tbrkrdvlIu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ParhkHP82a ++ cat /tmp/tmp.tbrkrdvlIu ++ rm /tmp/tmp.ParhkHP82a /tmp/tmp.tbrkrdvlIu ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/conf/smart-update.yml ++ mktemp + local LAST_OUT=/tmp/tmp.zpgsmTqWCO ++ mktemp + local LAST_ERR=/tmp/tmp.2GDLLAhJ5m + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/conf/smart-update.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zpgsmTqWCO perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.2GDLLAhJ5m + rm /tmp/tmp.zpgsmTqWCO /tmp/tmp.2GDLLAhJ5m + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.QGPB66Yx1E ++ mktemp + local LAST_ERR=/tmp/tmp.vAepfawWXH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QGPB66Yx1E persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted persistentvolumeclaim "proxydata-smart-update-proxysql-0" deleted persistentvolumeclaim "proxydata-smart-update-proxysql-1" deleted + cat /tmp/tmp.vAepfawWXH + rm /tmp/tmp.QGPB66Yx1E /tmp/tmp.vAepfawWXH + return 0 + desc 'Updating HAProxy PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- Updating HAProxy PXC cluster ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/conf/smart-update-haproxy.yml /tmp/tmp.g7q1LAQh2L/smart-update-haproxy.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1750-de45ff75"' /tmp/tmp.g7q1LAQh2L/smart-update-haproxy.yml + spinup_pxc smart-update /tmp/tmp.g7q1LAQh2L/smart-update-haproxy.yml + local cluster=smart-update + local config=/tmp/tmp.g7q1LAQh2L/smart-update-haproxy.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.IPlPgRyTz1 ++ mktemp + local LAST_ERR=/tmp/tmp.i8Ygk2l6pZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IPlPgRyTz1 secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.i8Ygk2l6pZ + rm /tmp/tmp.IPlPgRyTz1 /tmp/tmp.i8Ygk2l6pZ + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1750-de45ff75#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.W8PtDlm4La ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-3222~ + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.LuwjeGWZYM + local exit_status=0 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.W8PtDlm4La deployment.apps/pxc-client unchanged + cat /tmp/tmp.LuwjeGWZYM + rm /tmp/tmp.W8PtDlm4La /tmp/tmp.LuwjeGWZYM + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.g7q1LAQh2L/smart-update-haproxy.yml + '[' -z '' ']' + cat_config /tmp/tmp.g7q1LAQh2L/smart-update-haproxy.yml + kubectl_bin apply -f - + cat /tmp/tmp.g7q1LAQh2L/smart-update-haproxy.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1750-de45ff75#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.ooY9WNhYMg + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-3222~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.01FC8P8Wws + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ooY9WNhYMg perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.01FC8P8Wws + rm /tmp/tmp.ooY9WNhYMg /tmp/tmp.01FC8P8Wws + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.D8fv7F6ZG2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.B9xUH4dUTd +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.D8fv7F6ZG2 +++ cat /tmp/tmp.B9xUH4dUTd +++ rm /tmp/tmp.D8fv7F6ZG2 /tmp/tmp.B9xUH4dUTd +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3222 ++ mktemp + local LAST_OUT=/tmp/tmp.BrcGS7csc0 ++ mktemp + local LAST_ERR=/tmp/tmp.1y3Eswz7bE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3222 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3222 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3222 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.BrcGS7csc0 + cat /tmp/tmp.1y3Eswz7bE error: no matching resources found + rm /tmp/tmp.BrcGS7csc0 /tmp/tmp.1y3Eswz7bE + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met smart-update-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met smart-update-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met smart-update-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gz73E9LkSI +++ mktemp ++ local LAST_ERR=/tmp/tmp.tlzt5IjEgr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gz73E9LkSI ++ cat /tmp/tmp.tlzt5IjEgr ++ rm /tmp/tmp.gz73E9LkSI /tmp/tmp.tlzt5IjEgr ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.a29oTLds8T +++ mktemp ++ local LAST_ERR=/tmp/tmp.WWdI6Ps4nZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.a29oTLds8T ++ cat /tmp/tmp.WWdI6Ps4nZ ++ rm /tmp/tmp.a29oTLds8T /tmp/tmp.WWdI6Ps4nZ ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6CiWPLSQhA +++ mktemp ++ local LAST_ERR=/tmp/tmp.TYuaJ0HkXp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6CiWPLSQhA ++ cat /tmp/tmp.TYuaJ0HkXp ++ rm /tmp/tmp.6CiWPLSQhA /tmp/tmp.TYuaJ0HkXp ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sr89jgOtSn +++ mktemp ++ local LAST_ERR=/tmp/tmp.ezgyJV25Hd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sr89jgOtSn ++ cat /tmp/tmp.ezgyJV25Hd ++ rm /tmp/tmp.sr89jgOtSn /tmp/tmp.ezgyJV25Hd ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mGffvLBgzm +++ mktemp ++ local LAST_ERR=/tmp/tmp.FYu9h2TiEE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mGffvLBgzm ++ cat /tmp/tmp.FYu9h2TiEE ++ rm /tmp/tmp.mGffvLBgzm /tmp/tmp.FYu9h2TiEE ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ERKrCnTcMd +++ mktemp ++ local LAST_ERR=/tmp/tmp.T39TuGLhOm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ERKrCnTcMd ++ cat /tmp/tmp.T39TuGLhOm Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.ERKrCnTcMd /tmp/tmp.T39TuGLhOm ++ return 0 + '[' '' ']' ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pVYZT2JxO5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vnGqXKL7na +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.pVYZT2JxO5 +++ cat /tmp/tmp.vnGqXKL7na +++ rm /tmp/tmp.pVYZT2JxO5 /tmp/tmp.vnGqXKL7na +++ return 0 ++ client_pod=pxc-client-6644d8898f-pkgph ++ wait_pod pxc-client-6644d8898f-pkgph ++ local pod=pxc-client-6644d8898f-pkgph ++ local max_retry=480 ++ local ns= +++ echo pxc-client-6644d8898f-pkgph +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.MqgAkVL65F ++ mktemp + local LAST_ERR=/tmp/tmp.DUxUGqHOsw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MqgAkVL65F perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.DUxUGqHOsw + rm /tmp/tmp.MqgAkVL65F /tmp/tmp.DUxUGqHOsw + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update............................................ smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Uj2lbqIYQh +++ mktemp ++ local LAST_ERR=/tmp/tmp.JFNQVWhygn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Uj2lbqIYQh ++ cat /tmp/tmp.JFNQVWhygn ++ rm /tmp/tmp.Uj2lbqIYQh /tmp/tmp.JFNQVWhygn ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ukbuYHD76t +++ mktemp ++ local LAST_ERR=/tmp/tmp.RO7Ko9KXpE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ukbuYHD76t ++ cat /tmp/tmp.RO7Ko9KXpE ++ rm /tmp/tmp.ukbuYHD76t /tmp/tmp.RO7Ko9KXpE ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.M4WSLX9Xlm +++ mktemp ++ local LAST_ERR=/tmp/tmp.wcKpJUWNiI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.M4WSLX9Xlm ++ cat /tmp/tmp.wcKpJUWNiI ++ rm /tmp/tmp.M4WSLX9Xlm /tmp/tmp.wcKpJUWNiI ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jfTX8hvMaM +++ mktemp ++ local LAST_ERR=/tmp/tmp.BHi5ds7ewO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jfTX8hvMaM ++ cat /tmp/tmp.BHi5ds7ewO ++ rm /tmp/tmp.jfTX8hvMaM /tmp/tmp.BHi5ds7ewO ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hp4mlEK1tS +++ mktemp ++ local LAST_ERR=/tmp/tmp.t3Uw0x1uzE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hp4mlEK1tS ++ cat /tmp/tmp.t3Uw0x1uzE ++ rm /tmp/tmp.hp4mlEK1tS /tmp/tmp.t3Uw0x1uzE ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GHDqT8oIgY +++ mktemp ++ local LAST_ERR=/tmp/tmp.iDVhz7BMkc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GHDqT8oIgY ++ cat /tmp/tmp.iDVhz7BMkc ++ rm /tmp/tmp.GHDqT8oIgY /tmp/tmp.iDVhz7BMkc ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FilzIn9ewt +++ mktemp ++ local LAST_ERR=/tmp/tmp.hQ7lTdY7R6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FilzIn9ewt ++ cat /tmp/tmp.hQ7lTdY7R6 ++ rm /tmp/tmp.FilzIn9ewt /tmp/tmp.hQ7lTdY7R6 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VDgm8WUUCM +++ mktemp ++ local LAST_ERR=/tmp/tmp.CBiJ6GUuju ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VDgm8WUUCM ++ cat /tmp/tmp.CBiJ6GUuju ++ rm /tmp/tmp.VDgm8WUUCM /tmp/tmp.CBiJ6GUuju ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.l0BGGQ49tv +++ mktemp ++ local LAST_ERR=/tmp/tmp.FPUstcrTGi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.l0BGGQ49tv ++ cat /tmp/tmp.FPUstcrTGi ++ rm /tmp/tmp.l0BGGQ49tv /tmp/tmp.FPUstcrTGi ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jLz7C7eROK +++ mktemp ++ local LAST_ERR=/tmp/tmp.NckRi7aZLu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jLz7C7eROK ++ cat /tmp/tmp.NckRi7aZLu ++ rm /tmp/tmp.jLz7C7eROK /tmp/tmp.NckRi7aZLu ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.AGlfeYVGYc ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Hjjmge5hBV +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.AGlfeYVGYc +++++ cat /tmp/tmp.Hjjmge5hBV +++++ rm /tmp/tmp.AGlfeYVGYc /tmp/tmp.Hjjmge5hBV +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tcbt9N0m4x +++ mktemp ++ local LAST_ERR=/tmp/tmp.8YdJRCEVQE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tcbt9N0m4x ++ cat /tmp/tmp.8YdJRCEVQE ++ rm /tmp/tmp.tcbt9N0m4x /tmp/tmp.8YdJRCEVQE ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zyvZEi1Q4g +++ mktemp ++ local LAST_ERR=/tmp/tmp.IQnbKRYYa9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zyvZEi1Q4g ++ cat /tmp/tmp.IQnbKRYYa9 ++ rm /tmp/tmp.zyvZEi1Q4g /tmp/tmp.IQnbKRYYa9 ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Yt5x5Z4dLU +++ mktemp ++ local LAST_ERR=/tmp/tmp.QwUz58wXEm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Yt5x5Z4dLU ++ cat /tmp/tmp.QwUz58wXEm ++ rm /tmp/tmp.Yt5x5Z4dLU /tmp/tmp.QwUz58wXEm ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8m5BuetTZ5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.vFP7hUgM4D ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8m5BuetTZ5 ++ cat /tmp/tmp.vFP7hUgM4D ++ rm /tmp/tmp.8m5BuetTZ5 /tmp/tmp.vFP7hUgM4D ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + kubectl_bin delete -f /tmp/tmp.g7q1LAQh2L/smart-update-haproxy.yml ++ mktemp + local LAST_OUT=/tmp/tmp.oSIj83wyy4 ++ mktemp + local LAST_ERR=/tmp/tmp.M4vONLCiyq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.g7q1LAQh2L/smart-update-haproxy.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oSIj83wyy4 perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.M4vONLCiyq + rm /tmp/tmp.oSIj83wyy4 /tmp/tmp.M4vONLCiyq + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.zTsm7zSzXD ++ mktemp + local LAST_ERR=/tmp/tmp.5GIvqaV0pR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zTsm7zSzXD persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.5GIvqaV0pR + rm /tmp/tmp.zTsm7zSzXD /tmp/tmp.5GIvqaV0pR + return 0 + desc 'Updating PXC cluster with version service available but disabled' + set +o xtrace ----------------------------------------------------------------------------------- Updating PXC cluster with version service available but disabled ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/conf/smart-update-version-service-reachable.yml /tmp/tmp.g7q1LAQh2L/smart-update-version-service-reachable.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1750-de45ff75"' /tmp/tmp.g7q1LAQh2L/smart-update-version-service-reachable.yml + spinup_pxc smart-update /tmp/tmp.g7q1LAQh2L/smart-update-version-service-reachable.yml + local cluster=smart-update + local config=/tmp/tmp.g7q1LAQh2L/smart-update-version-service-reachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.qh6XxGwkhl ++ mktemp + local LAST_ERR=/tmp/tmp.a9bosTqx2t + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qh6XxGwkhl secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.a9bosTqx2t + rm /tmp/tmp.qh6XxGwkhl /tmp/tmp.a9bosTqx2t + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/client.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/client.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + local LAST_OUT=/tmp/tmp.MxwFhAjMHu + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1750-de45ff75#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.wtKvoUKhuZ + local exit_status=0 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-3222~ ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MxwFhAjMHu deployment.apps/pxc-client unchanged + cat /tmp/tmp.wtKvoUKhuZ + rm /tmp/tmp.MxwFhAjMHu /tmp/tmp.wtKvoUKhuZ + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.g7q1LAQh2L/smart-update-version-service-reachable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.g7q1LAQh2L/smart-update-version-service-reachable.yml + cat /tmp/tmp.g7q1LAQh2L/smart-update-version-service-reachable.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + local LAST_OUT=/tmp/tmp.Y7Aj6a4ulf + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1750-de45ff75#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_ERR=/tmp/tmp.dM1zcsOkBu + local exit_status=0 + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-3222~ ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Y7Aj6a4ulf perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.dM1zcsOkBu + rm /tmp/tmp.Y7Aj6a4ulf /tmp/tmp.dM1zcsOkBu + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5trrEos8PT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UTYu6E6d62 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.5trrEos8PT +++ cat /tmp/tmp.UTYu6E6d62 +++ rm /tmp/tmp.5trrEos8PT /tmp/tmp.UTYu6E6d62 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3222 ++ mktemp + local LAST_OUT=/tmp/tmp.Me2bTs9lOJ ++ mktemp + local LAST_ERR=/tmp/tmp.hX3ubORA5I + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3222 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3222 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-3222 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.Me2bTs9lOJ + cat /tmp/tmp.hX3ubORA5I error: no matching resources found + rm /tmp/tmp.Me2bTs9lOJ /tmp/tmp.hX3ubORA5I + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met smart-update-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met smart-update-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met smart-update-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eLaaR9OjEC +++ mktemp ++ local LAST_ERR=/tmp/tmp.UgFBNZRDBi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eLaaR9OjEC ++ cat /tmp/tmp.UgFBNZRDBi ++ rm /tmp/tmp.eLaaR9OjEC /tmp/tmp.UgFBNZRDBi ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RlcHkM9HDX +++ mktemp ++ local LAST_ERR=/tmp/tmp.2ViEsKHEcu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RlcHkM9HDX ++ cat /tmp/tmp.2ViEsKHEcu ++ rm /tmp/tmp.RlcHkM9HDX /tmp/tmp.2ViEsKHEcu ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.weKdrpj8MS +++ mktemp ++ local LAST_ERR=/tmp/tmp.xRQEtKukh4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.weKdrpj8MS ++ cat /tmp/tmp.xRQEtKukh4 ++ rm /tmp/tmp.weKdrpj8MS /tmp/tmp.xRQEtKukh4 ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xLPjQPpiJa +++ mktemp ++ local LAST_ERR=/tmp/tmp.lOjskG8WaJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xLPjQPpiJa ++ cat /tmp/tmp.lOjskG8WaJ ++ rm /tmp/tmp.xLPjQPpiJa /tmp/tmp.lOjskG8WaJ ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d7eLFXUe0o +++ mktemp ++ local LAST_ERR=/tmp/tmp.LJgVFsSMJ5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.d7eLFXUe0o ++ cat /tmp/tmp.LJgVFsSMJ5 ++ rm /tmp/tmp.d7eLFXUe0o /tmp/tmp.LJgVFsSMJ5 ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aECXsrPRw9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.BN7QwysHjE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aECXsrPRw9 ++ cat /tmp/tmp.BN7QwysHjE Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.aECXsrPRw9 /tmp/tmp.BN7QwysHjE ++ return 0 + '[' '' ']' ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CceVdUDtPs ++++ mktemp +++ local LAST_ERR=/tmp/tmp.dGsMmvYVOS +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.CceVdUDtPs +++ cat /tmp/tmp.dGsMmvYVOS +++ rm /tmp/tmp.CceVdUDtPs /tmp/tmp.dGsMmvYVOS +++ return 0 ++ client_pod=pxc-client-6644d8898f-pkgph ++ wait_pod pxc-client-6644d8898f-pkgph ++ local pod=pxc-client-6644d8898f-pkgph ++ local max_retry=480 ++ local ns= +++ echo pxc-client-6644d8898f-pkgph +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.6AtDB92u2e ++ mktemp + local LAST_ERR=/tmp/tmp.UB1G7O3Pf8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6AtDB92u2e perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.UB1G7O3Pf8 + rm /tmp/tmp.6AtDB92u2e /tmp/tmp.UB1G7O3Pf8 + return 0 + sleep 7 + desc 'check last pod to update' + set +o xtrace ----------------------------------------------------------------------------------- check last pod to update ----------------------------------------------------------------------------------- + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update.............................................. smart-update-pxc-0 is REALLY the last one to update + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zsGglMwZBx +++ mktemp ++ local LAST_ERR=/tmp/tmp.d6hGuoyBgL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zsGglMwZBx ++ cat /tmp/tmp.d6hGuoyBgL ++ rm /tmp/tmp.zsGglMwZBx /tmp/tmp.d6hGuoyBgL ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.A46MKrSnAL +++ mktemp ++ local LAST_ERR=/tmp/tmp.ySqHMDvcWT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A46MKrSnAL ++ cat /tmp/tmp.ySqHMDvcWT ++ rm /tmp/tmp.A46MKrSnAL /tmp/tmp.ySqHMDvcWT ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SoGAIl0jiO +++ mktemp ++ local LAST_ERR=/tmp/tmp.tq1DKuB4Xe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SoGAIl0jiO ++ cat /tmp/tmp.tq1DKuB4Xe ++ rm /tmp/tmp.SoGAIl0jiO /tmp/tmp.tq1DKuB4Xe ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8BDRUOOy5j +++ mktemp ++ local LAST_ERR=/tmp/tmp.o9jdq3KGiA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8BDRUOOy5j ++ cat /tmp/tmp.o9jdq3KGiA ++ rm /tmp/tmp.8BDRUOOy5j /tmp/tmp.o9jdq3KGiA ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PvMHdYWmBc +++ mktemp ++ local LAST_ERR=/tmp/tmp.pYuWEkib65 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PvMHdYWmBc ++ cat /tmp/tmp.pYuWEkib65 ++ rm /tmp/tmp.PvMHdYWmBc /tmp/tmp.pYuWEkib65 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.I1NLAqPn6E +++ mktemp ++ local LAST_ERR=/tmp/tmp.aCZOeBHrQ7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.I1NLAqPn6E ++ cat /tmp/tmp.aCZOeBHrQ7 ++ rm /tmp/tmp.I1NLAqPn6E /tmp/tmp.aCZOeBHrQ7 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MlpbatB0dV +++ mktemp ++ local LAST_ERR=/tmp/tmp.fmtGNSCzFp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MlpbatB0dV ++ cat /tmp/tmp.fmtGNSCzFp ++ rm /tmp/tmp.MlpbatB0dV /tmp/tmp.fmtGNSCzFp ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ySAzWD14ci +++ mktemp ++ local LAST_ERR=/tmp/tmp.glvuuVQqaK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ySAzWD14ci ++ cat /tmp/tmp.glvuuVQqaK ++ rm /tmp/tmp.ySAzWD14ci /tmp/tmp.glvuuVQqaK ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.g3QQMeIXcb +++ mktemp ++ local LAST_ERR=/tmp/tmp.P1d0spyCH6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.g3QQMeIXcb ++ cat /tmp/tmp.P1d0spyCH6 ++ rm /tmp/tmp.g3QQMeIXcb /tmp/tmp.P1d0spyCH6 ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yB0wepT8cE +++ mktemp ++ local LAST_ERR=/tmp/tmp.YJOijGps1h ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yB0wepT8cE ++ cat /tmp/tmp.YJOijGps1h ++ rm /tmp/tmp.yB0wepT8cE /tmp/tmp.YJOijGps1h ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.YTvoysYxp7 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.EPzQUb6b55 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.YTvoysYxp7 +++++ cat /tmp/tmp.EPzQUb6b55 +++++ rm /tmp/tmp.YTvoysYxp7 /tmp/tmp.EPzQUb6b55 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4yFL9mZpZI +++ mktemp ++ local LAST_ERR=/tmp/tmp.DMh8HaZvj4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4yFL9mZpZI ++ cat /tmp/tmp.DMh8HaZvj4 ++ rm /tmp/tmp.4yFL9mZpZI /tmp/tmp.DMh8HaZvj4 ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rdGhzKsQ53 +++ mktemp ++ local LAST_ERR=/tmp/tmp.8cOUhJZ8NG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rdGhzKsQ53 ++ cat /tmp/tmp.8cOUhJZ8NG ++ rm /tmp/tmp.rdGhzKsQ53 /tmp/tmp.8cOUhJZ8NG ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BToFZ9A7it +++ mktemp ++ local LAST_ERR=/tmp/tmp.PrQ6c55624 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BToFZ9A7it ++ cat /tmp/tmp.PrQ6c55624 ++ rm /tmp/tmp.BToFZ9A7it /tmp/tmp.PrQ6c55624 ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sYAUceIeFW +++ mktemp ++ local LAST_ERR=/tmp/tmp.fvYcBQDwPa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sYAUceIeFW ++ cat /tmp/tmp.fvYcBQDwPa ++ rm /tmp/tmp.sYAUceIeFW /tmp/tmp.fvYcBQDwPa ++ return 0 + client_pod=pxc-client-6644d8898f-pkgph + wait_pod pxc-client-6644d8898f-pkgph + local pod=pxc-client-6644d8898f-pkgph + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-pkgph ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-pkgph condition met pxc-client-6644d8898f-pkgph.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.g7q1LAQh2L/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.g7q1LAQh2L/select-1.sql + kubectl_bin delete -f /tmp/tmp.g7q1LAQh2L/smart-update-version-service-reachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.hvh5qvgtNG ++ mktemp + local LAST_ERR=/tmp/tmp.ZdqLMKpr0N + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.g7q1LAQh2L/smart-update-version-service-reachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hvh5qvgtNG perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.ZdqLMKpr0N + rm /tmp/tmp.hvh5qvgtNG /tmp/tmp.ZdqLMKpr0N + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.Fwcb2jK13O ++ mktemp + local LAST_ERR=/tmp/tmp.qTIOEDEpP9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Fwcb2jK13O persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.qTIOEDEpP9 + rm /tmp/tmp.Fwcb2jK13O /tmp/tmp.qTIOEDEpP9 + return 0 + desc cleanup + set +o xtrace ----------------------------------------------------------------------------------- cleanup ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.ywNuMAUJ8H ++ mktemp + local LAST_ERR=/tmp/tmp.FChj3hr8Xf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1750/e2e-tests/smart-update1/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ywNuMAUJ8H deployment.apps "version-service" deleted service "version-service" deleted + cat /tmp/tmp.FChj3hr8Xf + rm /tmp/tmp.ywNuMAUJ8H /tmp/tmp.FChj3hr8Xf + return 0 + destroy smart-update1-3222 + local namespace=smart-update1-3222 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v 'the object has been modified' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'get backup status: Job.batch' + tee /tmp/tmp.g7q1LAQh2L/operator.log + sort -u + grep -v level=info +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.kgS0xCvhBm +++ mktemp ++ local LAST_ERR=/tmp/tmp.hIEvbwGADJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kgS0xCvhBm ++ cat /tmp/tmp.hIEvbwGADJ ++ rm /tmp/tmp.kgS0xCvhBm /tmp/tmp.hIEvbwGADJ ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-7c5cdd5dd9-jpd4z ++ mktemp + local LAST_OUT=/tmp/tmp.wgaJ8cB0vr ++ mktemp + local LAST_ERR=/tmp/tmp.vpTb6UZ3nz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-7c5cdd5dd9-jpd4z + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wgaJ8cB0vr + cat /tmp/tmp.vpTb6UZ3nz + rm /tmp/tmp.wgaJ8cB0vr /tmp/tmp.vpTb6UZ3nz + return 0 2024-07-05T09:23:12.086Z INFO setup Manager starting up {"gitCommit": "de45ff75d55a19b5bd0fa01ec5862ada3c8f0243", "gitBranch": "PR-1750-de45ff75", "buildTime": "2024-07-05T07:35:25Z", "goVersion": "go1.22.5", "os": "linux", "arch": "amd64"} 2024-07-05T09:23:12.086Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.26.15-gke.1469001"} 2024-07-05T09:23:12.087Z INFO setup Registering Components. 2024-07-05T09:23:14.171Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2024-07-05T09:23:14.175Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2024-07-05T09:23:14.175Z INFO controller-runtime.metrics Starting metrics server 2024-07-05T09:23:14.175Z INFO controller-runtime.webhook Starting webhook server 2024-07-05T09:23:14.175Z INFO setup Starting the Cmd. 2024-07-05T09:23:14.175Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2024-07-05T09:23:14.176Z INFO controller-runtime.certwatcher Starting certificate watcher 2024-07-05T09:23:14.176Z INFO controller-runtime.certwatcher Updated current TLS certificate 2024-07-05T09:23:14.176Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2024-07-05T09:23:14.277Z INFO attempting to acquire leader lease pxc-operator/08db1feb.percona.com... 2024-07-05T09:23:14.294Z DEBUG events percona-xtradb-cluster-operator-7c5cdd5dd9-jpd4z_551bd3ba-f6f7-4f22-8e46-be0de8a93aad became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"d44da715-c92a-4317-939f-231908a9ec1d","apiVersion":"coordination.k8s.io/v1","resourceVersion":"55468"}, "reason": "LeaderElection"} 2024-07-05T09:23:14.294Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"} 2024-07-05T09:23:14.294Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2024-07-05T09:23:14.294Z INFO successfully acquired lease pxc-operator/08db1feb.percona.com 2024-07-05T09:23:14.295Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2024-07-05T09:23:14.295Z INFO Starting Controller {"controller": "pxc-controller"} 2024-07-05T09:23:14.295Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2024-07-05T09:23:14.295Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2024-07-05T09:23:14.510Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2024-07-05T09:23:14.510Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2024-07-05T09:23:14.510Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2024-07-05T09:25:39.559Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "c35f450f-5af0-43ce-b411-af82642f0608"} 2024-07-05T09:25:40.377Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "9f93f202-5722-4727-9bb6-8aa30cb68b6d"} 2024-07-05T09:25:45.306Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "3b5e74d1-ed86-4ec3-bcd3-962a7beced96"} 2024-07-05T09:25:50.458Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "7e20a046-7383-4d54-910f-8843420dfafb"} 2024-07-05T09:25:55.604Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f49ced2c-4831-46a8-9826-71e24508edf0"} 2024-07-05T09:26:00.814Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "dd3705c0-e67d-43d1-b48d-4b9642814b5f"} 2024-07-05T09:26:01.114Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "d9b3318f-7998-4c18-98f7-6e8a9eb6a268"} 2024-07-05T09:26:05.968Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0444ca9d-f544-494e-8673-af1d32e51c10"} 2024-07-05T09:26:11.272Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "946de77e-385a-4440-8771-79dd4f179ffc"} 2024-07-05T09:26:16.488Z INFO Waiting for ProxySQL to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "af8de6a9-7987-4b66-b997-0befdbddd9ff"} 2024-07-05T09:26:53.934Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "9f67ff35-3529-49ac-8f12-34aaca42e4f5", "user": "operator"} 2024-07-05T09:26:53.984Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "9f67ff35-3529-49ac-8f12-34aaca42e4f5", "user": "monitor"} 2024-07-05T09:26:54.075Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "9f67ff35-3529-49ac-8f12-34aaca42e4f5"} 2024-07-05T09:26:54.120Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "9f67ff35-3529-49ac-8f12-34aaca42e4f5"} 2024-07-05T09:26:54.161Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "9f67ff35-3529-49ac-8f12-34aaca42e4f5", "user": "xtrabackup"} 2024-07-05T09:26:54.236Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "9f67ff35-3529-49ac-8f12-34aaca42e4f5"} 2024-07-05T09:26:54.279Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "9f67ff35-3529-49ac-8f12-34aaca42e4f5", "user": "replication"} 2024-07-05T09:26:55.390Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "9f67ff35-3529-49ac-8f12-34aaca42e4f5", "err": "get primary pxc pod: not found"} 2024-07-05T09:27:00.012Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "71636a69-7719-4322-baab-fac3506f5e6d", "err": "get primary pxc pod: not found"} 2024-07-05T09:27:06.246Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "95a37c67-2cb4-4be6-8d5e-325fccdfd52f", "err": "get primary pxc pod: not found"} 2024-07-05T09:27:12.515Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e5b856e9-1644-42df-a5b2-387ca26669a0", "err": "get primary pxc pod: not found"} 2024-07-05T09:29:25.167Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "d9768310-c311-42c0-a30e-2c5015e07a11", "user": "root"} 2024-07-05T09:29:26.461Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "d9768310-c311-42c0-a30e-2c5015e07a11", "new version": "8.0.19-10"} 2024-07-05T09:29:27.731Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "d9768310-c311-42c0-a30e-2c5015e07a11"} 2024-07-05T09:29:32.615Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "5ac59c55-3682-4170-9a1b-37b8a31caa34"} 2024-07-05T09:29:39.535Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "ff3d14dd-f05c-426a-bb4a-571b9be6aaea"} 2024-07-05T09:29:45.642Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "8563c1d2-7abd-470f-b79e-98f8a9678ed3"} 2024-07-05T09:29:52.100Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0acafb40-8120-4584-a766-2e878c03e61b"} 2024-07-05T09:29:58.565Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "a672e474-40ca-4e69-9ba5-562bd687002e"} 2024-07-05T09:30:05.083Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "64311f39-bab7-41c9-a2c2-c9f8a154068f"} 2024-07-05T09:30:11.144Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "53dea807-822e-4299-86fc-e7629117a188"} 2024-07-05T09:30:17.475Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "06e574a9-2532-4c30-bb38-fa41fa951564"} 2024-07-05T09:30:24.430Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "3ddccbad-caac-45c5-8eae-779e67ad3df1"} 2024-07-05T09:30:30.777Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "33b7e1f2-f67d-488b-b447-9363d95e8a13"} 2024-07-05T09:30:37.338Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "d80e42b5-2844-49fc-8c45-48a6e3836daf"} 2024-07-05T09:30:38.073Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96"} 2024-07-05T09:30:38.082Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-2"} 2024-07-05T09:30:38.082Z INFO primary pod {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-0.smart-update-pxc.smart-update1-3222.svc.cluster.local"} 2024-07-05T09:30:58.349Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-2", "reason": "PodInitializing"} 2024-07-05T09:32:08.411Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-2"} 2024-07-05T09:32:58.458Z INFO pod present in hostgroups {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-2"} 2024-07-05T09:32:58.466Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-1"} 2024-07-05T09:32:58.466Z INFO pod is online {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-2"} 2024-07-05T09:33:18.704Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-1", "reason": "PodInitializing"} 2024-07-05T09:34:28.739Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-1"} 2024-07-05T09:35:38.766Z INFO pod present in hostgroups {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-1"} 2024-07-05T09:35:38.773Z INFO apply changes to primary pod {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-0"} 2024-07-05T09:35:38.773Z INFO pod is online {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-1"} 2024-07-05T09:35:59.030Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-0", "reason": "PodInitializing"} 2024-07-05T09:36:59.057Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-0"} 2024-07-05T09:37:59.085Z INFO pod present in hostgroups {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-0"} 2024-07-05T09:37:59.092Z INFO pod is online {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "pod name": "smart-update-pxc-0"} 2024-07-05T09:37:59.092Z INFO smart update finished {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96"} 2024-07-05T09:37:59.187Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0bbf4150-1f58-442a-a429-e59b42033d96", "new version": "8.0.36-28.1"} 2024-07-05T09:38:03.503Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "627e63ce-49e7-42da-a886-1ee7d3dcf736"} 2024-07-05T09:38:07.281Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "b49a9c76-3a53-4a31-846b-6611e91f2e6c"} 2024-07-05T09:38:13.649Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "eeb51606-dd37-4449-a019-945532358a5c"} 2024-07-05T09:38:19.995Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "aa73436f-80f3-4def-993a-65311144422c"} 2024-07-05T09:38:26.402Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "b753721b-a8b1-4563-938e-e39db5319443"} 2024-07-05T09:38:32.931Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "36d1aa94-3e26-437e-afbc-4cab94022a5c"} 2024-07-05T09:39:11.161Z ERROR Update status {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "435b539b-f353-43cd-813f-f5c5dda8b893", "error": "write status: PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found", "errorVerbose": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found\nwrite status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:157\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:140\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:201\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:245\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-05T09:39:28.385Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "813871eb-a65d-46d5-9c94-ee0c3a6bf491"} 2024-07-05T09:39:29.112Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "045e84a2-4f61-49f9-9cc8-51242e4be661"} 2024-07-05T09:39:29.347Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: bb9f83ea-e5e3-4f0c-985b-c6ef9c32c374 2024-07-05T09:39:29.553Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "d3b46e90-3a84-4887-9d6a-bc52036e0c25"} 2024-07-05T09:39:29.693Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "76bf5662-2451-4055-8bb3-4bff8c250aaa"} 2024-07-05T09:39:34.540Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f0d2c624-4725-4d29-af07-2969d2d317f4"} 2024-07-05T09:39:39.713Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "4c6e4801-e761-4948-b6bb-5fe31ca0fd9e"} 2024-07-05T09:39:44.862Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "14048c01-0039-41d8-8acf-431a80f75cd2"} 2024-07-05T09:39:50.033Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e22d7500-996e-4cbf-a825-641e4dd0a40b"} 2024-07-05T09:39:55.178Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "3b60bf49-6eb5-4e8c-8325-a835a4619bb1"} 2024-07-05T09:40:00.336Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "35c9fd37-d890-4a80-a916-4c3804851630"} 2024-07-05T09:40:05.501Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "60843ac1-c74d-4718-ad6d-a9ac446adf19"} 2024-07-05T09:40:10.636Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "7a9157d7-3c64-40f3-8eeb-296003cb799d"} 2024-07-05T09:40:15.808Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e5f0bf8c-6807-4e37-9159-0bacbc21fe40"} 2024-07-05T09:40:20.961Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "5212a817-6821-470f-9da4-213a6e15646b"} 2024-07-05T09:40:26.105Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "c6142d21-9317-4ce7-9bba-f8225a0664ae"} 2024-07-05T09:40:31.301Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "1df04d49-077d-4d3c-9774-dec6d19ea624"} 2024-07-05T09:40:36.792Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0e9379be-fde2-4794-9707-946b3c596a26"} 2024-07-05T09:40:42.071Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "fd4e354b-c646-4362-8374-8c8cfa44b4c7"} 2024-07-05T09:40:47.498Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "c36e923f-d5b0-40e2-ae52-279279ddff0f"} 2024-07-05T09:40:47.840Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "61c49f70-5f62-4917-b58d-46e74c3b3d91"} 2024-07-05T09:40:50.094Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "edeb0f3a-6741-44e7-a4ca-aa6ae362f096"} 2024-07-05T09:40:52.868Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "afeb0bcb-477e-4249-a9ef-374b2d0e82ea"} 2024-07-05T09:40:58.175Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "2c58f438-6977-46e3-8d03-db3100f05f5f"} 2024-07-05T09:41:03.462Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "3e73b515-7165-44a0-b66c-51af9eba7588"} 2024-07-05T09:41:08.781Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "a8698ef7-1114-45de-b79a-9fc839227d0d"} 2024-07-05T09:41:14.012Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "c68b529b-b481-4598-9f9d-37fcafe9cf63"} 2024-07-05T09:41:19.363Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "445ed393-8347-4dcf-8e6b-e3bb9517a8b2"} 2024-07-05T09:43:14.062Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "01eb2def-9134-4708-92c9-f118007c8380", "new version": "8.0.19-10"} 2024-07-05T09:44:25.079Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e8123bbe-b157-4eaf-a240-3a3f21febb04"} 2024-07-05T09:44:25.092Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e8123bbe-b157-4eaf-a240-3a3f21febb04", "pod name": "smart-update-pxc-2"} 2024-07-05T09:44:25.092Z INFO primary pod {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e8123bbe-b157-4eaf-a240-3a3f21febb04", "pod name": "smart-update-pxc-0.smart-update-pxc.smart-update1-3222"} 2024-07-05T09:44:45.377Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e8123bbe-b157-4eaf-a240-3a3f21febb04", "pod name": "smart-update-pxc-2", "reason": "PodInitializing"} 2024-07-05T09:45:45.412Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e8123bbe-b157-4eaf-a240-3a3f21febb04", "pod name": "smart-update-pxc-2"} 2024-07-05T09:45:45.434Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e8123bbe-b157-4eaf-a240-3a3f21febb04", "pod name": "smart-update-pxc-1"} 2024-07-05T09:46:05.634Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e8123bbe-b157-4eaf-a240-3a3f21febb04", "pod name": "smart-update-pxc-1", "reason": "PodInitializing"} 2024-07-05T09:47:15.696Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e8123bbe-b157-4eaf-a240-3a3f21febb04", "pod name": "smart-update-pxc-1"} 2024-07-05T09:47:15.715Z INFO apply changes to primary pod {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e8123bbe-b157-4eaf-a240-3a3f21febb04", "pod name": "smart-update-pxc-0"} 2024-07-05T09:47:35.934Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e8123bbe-b157-4eaf-a240-3a3f21febb04", "pod name": "smart-update-pxc-0", "reason": "PodInitializing"} 2024-07-05T09:48:36.124Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e8123bbe-b157-4eaf-a240-3a3f21febb04", "pod name": "smart-update-pxc-0"} 2024-07-05T09:48:36.142Z INFO smart update finished {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e8123bbe-b157-4eaf-a240-3a3f21febb04"} 2024-07-05T09:48:36.222Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "e8123bbe-b157-4eaf-a240-3a3f21febb04", "new version": "8.0.36-28.1"} 2024-07-05T09:49:09.724Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "8fe5211f-96a0-4a57-83c9-9ce5ff92abe0"} 2024-07-05T09:49:09.979Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "579aa890-40ca-487e-9c2b-178d28fa8db6"} 2024-07-05T09:49:10.719Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "37b45b4e-b5dd-4f7b-8acd-ba013d797222"} 2024-07-05T09:49:10.976Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "5a8b8f5d-5171-4984-8e0b-3219ac5181d7"} 2024-07-05T09:49:40.780Z ERROR Update status {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "bb848c5a-6e94-4016-b90b-13707a928b73", "error": "write status: PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found", "errorVerbose": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found\nwrite status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:157\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:140\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:201\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:245\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-05T09:49:58.889Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "42f671c0-bd93-430f-b541-8e2cf49ee763"} 2024-07-05T09:49:59.618Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "482e19ae-a9eb-4c40-9106-c2d1612bc2ba"} 2024-07-05T09:49:59.807Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 3798679f-8ebc-4d6e-afef-7fab18ae26d1 2024-07-05T09:50:00.015Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "272e49b0-b729-46b8-9ce7-e0583510d128"} 2024-07-05T09:50:05.010Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "2a38834a-3729-447a-814c-fcd0de8f40b6"} 2024-07-05T09:50:10.219Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "05b0d9c7-3aef-440f-b421-01f93c027501"} 2024-07-05T09:50:15.390Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "256e4c87-ebaa-42c9-9f5e-b5a7b1cb2ab5"} 2024-07-05T09:50:20.524Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "1de5ff07-676a-4c0a-8999-cb07f2face4b"} 2024-07-05T09:50:25.696Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "982555ba-bb6c-4b67-b0ca-22f4e3f51396"} 2024-07-05T09:50:30.925Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "90dc3ad3-63ca-4265-aac2-f46fd9bd071d"} 2024-07-05T09:50:36.073Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "6b069e59-54f8-43e6-a5c6-c4090375e5ba"} 2024-07-05T09:50:41.245Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "a7a1eb74-b437-4ae9-8c99-87762b20a494"} 2024-07-05T09:50:46.418Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "cbd58072-9b79-4a99-812f-c06a76057ccf"} 2024-07-05T09:50:51.881Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "594f3496-736b-4140-a6d2-aaeaa3fbe896"} 2024-07-05T09:50:57.138Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "8da9222e-d617-43c2-8dae-257064657e45"} 2024-07-05T09:51:02.565Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f363f6a2-39f0-4c61-bb16-0560787e34d5"} 2024-07-05T09:51:07.794Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "8336fa1e-c60e-4949-b02e-91a1e4be0f67"} 2024-07-05T09:51:12.978Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "c8c0b089-9e26-4fa2-8c12-8a081782c449"} 2024-07-05T09:51:13.212Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "113f49c8-46fc-4511-a2f1-4a2f50d08e6a"} 2024-07-05T09:51:15.223Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "113f49c8-46fc-4511-a2f1-4a2f50d08e6a", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.132.158.88:3306: connect: connection refused"} 2024-07-05T09:51:18.209Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "a1695b0e-f948-40b2-93c2-ffcd759c1843"} 2024-07-05T09:51:18.600Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "ebf4ef69-2baa-413f-854f-17515332d855"} 2024-07-05T09:51:23.539Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "fbb8392f-be31-4b0a-8e30-8c1966252149"} 2024-07-05T09:51:28.774Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "94dd3e54-52f1-441a-8889-a03a71a826c2"} 2024-07-05T09:51:33.991Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "0cfeec48-fa69-497d-b1b9-6c1081123924"} 2024-07-05T09:51:39.291Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "aaf22ccf-b846-45a6-bba2-1536d2c56266"} 2024-07-05T09:53:46.419Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f18b933d-d877-4dff-88c1-b083b184487d", "new version": "8.0.19-10"} 2024-07-05T09:54:51.054Z INFO statefulSet was changed, run smart update {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f4cf4a69-303f-4adb-899b-59c0622bbe9f"} 2024-07-05T09:54:51.066Z INFO primary pod {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f4cf4a69-303f-4adb-899b-59c0622bbe9f", "pod name": "smart-update-pxc-0.smart-update-pxc.smart-update1-3222"} 2024-07-05T09:54:51.067Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f4cf4a69-303f-4adb-899b-59c0622bbe9f", "pod name": "smart-update-pxc-2"} 2024-07-05T09:55:11.354Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f4cf4a69-303f-4adb-899b-59c0622bbe9f", "pod name": "smart-update-pxc-2", "reason": "PodInitializing"} 2024-07-05T09:56:11.372Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f4cf4a69-303f-4adb-899b-59c0622bbe9f", "pod name": "smart-update-pxc-2"} 2024-07-05T09:56:11.392Z INFO apply changes to secondary pod {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f4cf4a69-303f-4adb-899b-59c0622bbe9f", "pod name": "smart-update-pxc-1"} 2024-07-05T09:56:31.613Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f4cf4a69-303f-4adb-899b-59c0622bbe9f", "pod name": "smart-update-pxc-1", "reason": "PodInitializing"} 2024-07-05T09:57:41.664Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f4cf4a69-303f-4adb-899b-59c0622bbe9f", "pod name": "smart-update-pxc-1"} 2024-07-05T09:57:41.681Z INFO apply changes to primary pod {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f4cf4a69-303f-4adb-899b-59c0622bbe9f", "pod name": "smart-update-pxc-0"} 2024-07-05T09:58:01.922Z INFO pod is waiting {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f4cf4a69-303f-4adb-899b-59c0622bbe9f", "pod name": "smart-update-pxc-0", "reason": "PodInitializing"} 2024-07-05T09:59:11.946Z INFO pod is running {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f4cf4a69-303f-4adb-899b-59c0622bbe9f", "pod name": "smart-update-pxc-0"} 2024-07-05T09:59:11.968Z INFO smart update finished {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f4cf4a69-303f-4adb-899b-59c0622bbe9f"} 2024-07-05T09:59:12.074Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "f4cf4a69-303f-4adb-899b-59c0622bbe9f", "new version": "8.0.36-28.1"} 2024-07-05T10:00:06.134Z ERROR Update status {"controller": "pxc-controller", "namespace": "smart-update1-3222", "name": "smart-update", "reconcileID": "c0dee2cc-e875-4fae-93eb-5b24f217ed6f", "error": "write status: PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found", "errorVerbose": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found\nwrite status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:157\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:140\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:201\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:245\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:203 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:245 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.3Vv0iLrwer ++ mktemp + local LAST_ERR=/tmp/tmp.RX2kKK7yX4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3Vv0iLrwer No resources found + cat /tmp/tmp.RX2kKK7yX4 + rm /tmp/tmp.3Vv0iLrwer /tmp/tmp.RX2kKK7yX4 + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.YovVPOMmH6 ++ mktemp + local LAST_ERR=/tmp/tmp.8kAF8kECK2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YovVPOMmH6 No resources found + cat /tmp/tmp.8kAF8kECK2 + rm /tmp/tmp.YovVPOMmH6 /tmp/tmp.8kAF8kECK2 + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.xg4WoUKFlC ++ mktemp + local LAST_ERR=/tmp/tmp.loeD7Y2sQE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xg4WoUKFlC No resources found + cat /tmp/tmp.loeD7Y2sQE + rm /tmp/tmp.xg4WoUKFlC /tmp/tmp.loeD7Y2sQE + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.7zZE1DPwcS ++ mktemp + local LAST_ERR=/tmp/tmp.Vf950p7mg5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7zZE1DPwcS validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.Vf950p7mg5 + rm /tmp/tmp.7zZE1DPwcS /tmp/tmp.Vf950p7mg5 + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace smart-update1-3222 + rm -rf /tmp/tmp.g7q1LAQh2L + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ mktemp + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.0XPLNMA8pT + local LAST_OUT=/tmp/tmp.cS36EXt2IP ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.4hr4jbESvx + local LAST_ERR=/tmp/tmp.ppL5WeUNSE + local exit_status=0 + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + for i in '$(seq 0 2)' + kubectl delete --grace-period=0 --force=true namespace smart-update1-3222 + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator