Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/logs/smart-update1-8-0.log
WARNING: version difference between client (1.30) and server (1.27) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.27) exceeds the supported minor version skew of +/-1
+ API=pxc.percona.com/v9-9-9
+ TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ CLUSTER=smart-update
+ CLUSTER_SIZE=3
+ PROXY_SIZE=2
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]]
++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0
++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/'
+ PXC_VER=8.0
+ TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ VS_URL=http://version-service
+ VS_PORT=11000
+ VS_ENDPOINT=http://version-service:11000
+ main
+ create_infra smart-update1-13135
+ local ns=smart-update1-13135
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n smart-update1-19052 smart-update --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.pl6xnojCiL
++ mktemp
+ local LAST_ERR=/tmp/tmp.v28xrF8nLb
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.pl6xnojCiL
perconaxtradbcluster.pxc.percona.com "smart-update" deleted
+ cat /tmp/tmp.v28xrF8nLb
+ rm /tmp/tmp.pl6xnojCiL /tmp/tmp.v28xrF8nLb
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.J8oqHfwiIr
++ mktemp
+ local LAST_ERR=/tmp/tmp.2Rt09GEqSE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.J8oqHfwiIr
No resources found
+ cat /tmp/tmp.2Rt09GEqSE
+ rm /tmp/tmp.J8oqHfwiIr /tmp/tmp.2Rt09GEqSE
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.RhWgclA0rk
++ mktemp
+ local LAST_ERR=/tmp/tmp.vY5AUcE6da
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.RhWgclA0rk
No resources found
+ cat /tmp/tmp.vY5AUcE6da
+ rm /tmp/tmp.RhWgclA0rk /tmp/tmp.vY5AUcE6da
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ xargs kubectl delete ns
++ mktemp
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ local LAST_OUT=/tmp/tmp.qX2GdzZCDg
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.y2FxnP387W
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.vPLHnZ2N4n
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.Ylc4hV9mLh
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.y2FxnP387W
+ cat /tmp/tmp.Ylc4hV9mLh
+ rm /tmp/tmp.y2FxnP387W /tmp/tmp.Ylc4hV9mLh
+ return 0
namespace "cert-manager" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
namespace "smart-update1-19052" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.qX2GdzZCDg
namespace "pxc-operator" deleted
+ cat /tmp/tmp.vPLHnZ2N4n
+ rm /tmp/tmp.qX2GdzZCDg /tmp/tmp.vPLHnZ2N4n
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'namespace/pxc-operator - '
namespace/pxc-operator - + set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.GIGYuPZViv
++ mktemp
+ local LAST_ERR=/tmp/tmp.PynjfzauoU
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.GIGYuPZViv
namespace/pxc-operator created
+ cat /tmp/tmp.PynjfzauoU
+ rm /tmp/tmp.GIGYuPZViv /tmp/tmp.PynjfzauoU
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5s6ght2cfj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hUlXt0x68B
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.5s6ght2cfj
++ cat /tmp/tmp.hUlXt0x68B
++ rm /tmp/tmp.5s6ght2cfj /tmp/tmp.hUlXt0x68B
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1785-7e3ddd30-4-cluster5 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.vDjeWTDTHB
++ mktemp
+ local LAST_ERR=/tmp/tmp.AR4CZfs7Bc
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1785-7e3ddd30-4-cluster5 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vDjeWTDTHB
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1785-7e3ddd30-4-cluster5" modified.
+ cat /tmp/tmp.AR4CZfs7Bc
+ rm /tmp/tmp.vDjeWTDTHB /tmp/tmp.AR4CZfs7Bc
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.BgmzSpBKxp
++ mktemp
+ local LAST_ERR=/tmp/tmp.mf5ur4TJUv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.BgmzSpBKxp
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.mf5ur4TJUv
+ rm /tmp/tmp.BgmzSpBKxp /tmp/tmp.mf5ur4TJUv
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/deploy/cw-rbac.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.rFIpdjGarR
++ mktemp
+ local LAST_ERR=/tmp/tmp.BQOKKkeFI6
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.rFIpdjGarR
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.BQOKKkeFI6
+ rm /tmp/tmp.rFIpdjGarR /tmp/tmp.BQOKKkeFI6
+ return 0
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/deploy/cw-operator.yaml
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1785-7e3ddd30^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' -
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.VXDen6K6vP
++ mktemp
+ local LAST_ERR=/tmp/tmp.M6AbSjGxSP
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.VXDen6K6vP
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.M6AbSjGxSP
+ rm /tmp/tmp.VXDen6K6vP /tmp/tmp.M6AbSjGxSP
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.VsWzACyQKZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.gZ0Pm1jM2O
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.VsWzACyQKZ
pod/percona-xtradb-cluster-operator-66cc8ffb95-nb2nj condition met
+ cat /tmp/tmp.gZ0Pm1jM2O
+ rm /tmp/tmp.VsWzACyQKZ /tmp/tmp.gZ0Pm1jM2O
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0JTht1tEt2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zx6Gl0Faaj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0JTht1tEt2
++ cat /tmp/tmp.zx6Gl0Faaj
++ rm /tmp/tmp.0JTht1tEt2 /tmp/tmp.zx6Gl0Faaj
++ return 0
+ wait_pod percona-xtradb-cluster-operator-66cc8ffb95-nb2nj 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-66cc8ffb95-nb2nj
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-66cc8ffb95-nb2nj
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-66cc8ffb95-nb2nj condition met
percona-xtradb-cluster-operator-66cc8ffb95-nb2nj.Ok
+ sleep 3
+ create_namespace smart-update1-13135
+ local namespace=smart-update1-13135
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl api-resources
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
++ mktemp
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces smart-update1-13135'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces smart-update1-13135
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace smart-update1-13135
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ local LAST_OUT=/tmp/tmp.y1giElnccP
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.vQgk0T2yeP
+ local LAST_ERR=/tmp/tmp.uwOMesCVfa
+ local exit_status=0
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.mHIV0ofQy7
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace smart-update1-13135
+ awk '{print$1}'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace smart-update1-13135
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.y1giElnccP
+ cat /tmp/tmp.uwOMesCVfa
+ rm /tmp/tmp.y1giElnccP /tmp/tmp.uwOMesCVfa
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace smart-update1-13135
namespace "gmp-public" deleted
namespace "gmp-system" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.vQgk0T2yeP
+ cat /tmp/tmp.mHIV0ofQy7
Error from server (NotFound): namespaces "smart-update1-13135" not found
+ rm /tmp/tmp.vQgk0T2yeP /tmp/tmp.mHIV0ofQy7
+ return 1
+ :
+ wait_for_delete namespace/smart-update1-13135
+ local res=namespace/smart-update1-13135
+ echo -n 'namespace/smart-update1-13135 - '
namespace/smart-update1-13135 - + set +o xtrace
Error from server (NotFound): namespaces "smart-update1-13135" not found
+ desc 'create namespace smart-update1-13135'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace smart-update1-13135
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace smart-update1-13135
++ mktemp
+ local LAST_OUT=/tmp/tmp.gPvXzUFx7t
++ mktemp
+ local LAST_ERR=/tmp/tmp.dgegrLZd4q
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace smart-update1-13135
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gPvXzUFx7t
namespace/smart-update1-13135 created
+ cat /tmp/tmp.dgegrLZd4q
+ rm /tmp/tmp.gPvXzUFx7t /tmp/tmp.dgegrLZd4q
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RFO83XjjUi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ki0vOtpNXM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RFO83XjjUi
++ cat /tmp/tmp.ki0vOtpNXM
++ rm /tmp/tmp.RFO83XjjUi /tmp/tmp.ki0vOtpNXM
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1785-7e3ddd30-4-cluster5 --namespace=smart-update1-13135
++ mktemp
+ local LAST_OUT=/tmp/tmp.WWdPpHKTGh
++ mktemp
+ local LAST_ERR=/tmp/tmp.Kn6uVjmVRO
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1785-7e3ddd30-4-cluster5 --namespace=smart-update1-13135
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.WWdPpHKTGh
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1785-7e3ddd30-4-cluster5" modified.
+ cat /tmp/tmp.Kn6uVjmVRO
+ rm /tmp/tmp.WWdPpHKTGh /tmp/tmp.Kn6uVjmVRO
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.zSM1lctWbt
++ mktemp
+ local LAST_ERR=/tmp/tmp.KjTte9TPSg
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.zSM1lctWbt
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.KjTte9TPSg
+ rm /tmp/tmp.zSM1lctWbt /tmp/tmp.KjTte9TPSg
+ return 0
+ deploy_version_service
+ desc 'install version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install version service
-----------------------------------------------------------------------------------

+ kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json
++ mktemp
+ local LAST_OUT=/tmp/tmp.jiQsi2Q2bf
++ mktemp
+ local LAST_ERR=/tmp/tmp.TBN5z2Li1y
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.jiQsi2Q2bf
configmap/versions created
+ cat /tmp/tmp.TBN5z2Li1y
+ rm /tmp/tmp.jiQsi2Q2bf /tmp/tmp.TBN5z2Li1y
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/conf/vs.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.5baEtXJ6Sr
++ mktemp
+ local LAST_ERR=/tmp/tmp.7rCnio6dGk
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/conf/vs.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5baEtXJ6Sr
deployment.apps/version-service created
service/version-service created
+ cat /tmp/tmp.7rCnio6dGk
+ rm /tmp/tmp.5baEtXJ6Sr /tmp/tmp.7rCnio6dGk
+ return 0
+ sleep 10
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.Zl9Tmw8GLh
++ mktemp
+ local LAST_ERR=/tmp/tmp.JUdlT1RD4n
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Zl9Tmw8GLh
namespace/cert-manager created
+ cat /tmp/tmp.JUdlT1RD4n
+ rm /tmp/tmp.Zl9Tmw8GLh /tmp/tmp.JUdlT1RD4n
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.b31tcLKOrz
++ mktemp
+ local LAST_ERR=/tmp/tmp.HVO2jyjmcK
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.b31tcLKOrz
namespace/cert-manager labeled
+ cat /tmp/tmp.HVO2jyjmcK
+ rm /tmp/tmp.b31tcLKOrz /tmp/tmp.HVO2jyjmcK
+ return 0
+ kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.Px28VwLDeZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.ebmGLJRcjm
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Px28VwLDeZ
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews configured
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection configured
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.ebmGLJRcjm
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.Px28VwLDeZ /tmp/tmp.ebmGLJRcjm
+ return 0
+ '[' '' == 4.10 ']'
+ sleep 70
++ jq -r '.versions[].matrix.pxc[].imagePath'
++ grep :8.0
+++ get_operator_pod
+++ local label_prefix=app.kubernetes.io/
++++ grep -c percona-xtradb-cluster-operator
++ head -n1
++ tail -n3
++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ sort -V
+++ local check_label=1
+++ [[ 1 -eq 0 ]]
+++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ZLOjTVzGhl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9j5KJlbkzh
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.ZLOjTVzGhl
+++ cat /tmp/tmp.9j5KJlbkzh
+++ rm /tmp/tmp.ZLOjTVzGhl /tmp/tmp.9j5KJlbkzh
+++ return 0
++ kubectl_bin exec -ti percona-xtradb-cluster-operator-66cc8ffb95-nb2nj -n pxc-operator -- curl -s http://version-service.smart-update1-13135.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
+++ mktemp
++ local LAST_OUT=/tmp/tmp.no3xdAaWRe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zy187bOYNl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -ti percona-xtradb-cluster-operator-66cc8ffb95-nb2nj -n pxc-operator -- curl -s http://version-service.smart-update1-13135.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.no3xdAaWRe
++ cat /tmp/tmp.zy187bOYNl
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.no3xdAaWRe /tmp/tmp.zy187bOYNl
++ return 0
+ IMAGE_PXC=percona/percona-xtradb-cluster:8.0.19-10.1
+ desc 'patch crd'
+ set +o xtrace


-----------------------------------------------------------------------------------
patch crd
-----------------------------------------------------------------------------------

+ kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.W64t5WapVl
++ mktemp
+ local LAST_ERR=/tmp/tmp.7NlPVbCM0W
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.W64t5WapVl
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched
+ cat /tmp/tmp.7NlPVbCM0W
+ rm /tmp/tmp.W64t5WapVl /tmp/tmp.7NlPVbCM0W
+ return 0
+ desc 'Updating ProxySQL PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Updating ProxySQL PXC cluster
-----------------------------------------------------------------------------------

+ cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/conf/smart-update.yml /tmp/tmp.c7lkEUIUkH/smart-update.yml
+ yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1785-7e3ddd30"' /tmp/tmp.c7lkEUIUkH/smart-update.yml
+ spinup_pxc smart-update /tmp/tmp.c7lkEUIUkH/smart-update.yml
+ local cluster=smart-update
+ local config=/tmp/tmp.c7lkEUIUkH/smart-update.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.EKaoonoy40
++ mktemp
+ local LAST_ERR=/tmp/tmp.eYoDxliWrh
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EKaoonoy40
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.eYoDxliWrh
+ rm /tmp/tmp.EKaoonoy40 /tmp/tmp.eYoDxliWrh
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1785-7e3ddd30#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-13135~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ local LAST_OUT=/tmp/tmp.6vVl5DQqGS
++ mktemp
+ local LAST_ERR=/tmp/tmp.CunUKIyluG
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6vVl5DQqGS
deployment.apps/pxc-client created
+ cat /tmp/tmp.CunUKIyluG
+ rm /tmp/tmp.6vVl5DQqGS /tmp/tmp.CunUKIyluG
+ return 0
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ apply_config /tmp/tmp.c7lkEUIUkH/smart-update.yml
+ '[' -z '' ']'
+ cat_config /tmp/tmp.c7lkEUIUkH/smart-update.yml
+ kubectl_bin apply -f -
+ cat /tmp/tmp.c7lkEUIUkH/smart-update.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1785-7e3ddd30#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ local LAST_OUT=/tmp/tmp.6yGSRl6kRb
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
++ mktemp
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-13135~
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ local LAST_ERR=/tmp/tmp.Zt9Ae1piCk
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6yGSRl6kRb
perconaxtradbcluster.pxc.percona.com/smart-update created
+ cat /tmp/tmp.Zt9Ae1piCk
+ rm /tmp/tmp.6yGSRl6kRb /tmp/tmp.Zt9Ae1piCk
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.RgKxJyRbUh
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xGEE7AmwKd
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.RgKxJyRbUh
+++ cat /tmp/tmp.xGEE7AmwKd
+++ rm /tmp/tmp.RgKxJyRbUh /tmp/tmp.xGEE7AmwKd
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.UDDvgZJVE8
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.pAXatk6GFA
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.UDDvgZJVE8
+++ cat /tmp/tmp.pAXatk6GFA
+++ rm /tmp/tmp.UDDvgZJVE8 /tmp/tmp.pAXatk6GFA
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-proxysql
++ return
+ local proxy=smart-update-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-13135
++ mktemp
+ local LAST_OUT=/tmp/tmp.xWtsLvSSxE
++ mktemp
+ local LAST_ERR=/tmp/tmp.o0ks5uBRVN
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-13135
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-13135
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-13135
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.xWtsLvSSxE
+ cat /tmp/tmp.o0ks5uBRVN
error: no matching resources found
+ rm /tmp/tmp.xWtsLvSSxE /tmp/tmp.o0ks5uBRVN
+ return 1
+ true
+ wait_for_running smart-update-proxysql 1
+ local name=smart-update-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-proxysql-0 480
+ local pod=smart-update-proxysql-0
+ local max_retry=480
+ local ns=
++ echo smart-update-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
Error from server (NotFound): pods "smart-update-proxysql-0" not found
smart-update-proxysql-0...........Ok
+ wait_for_running smart-update-pxc 3
+ local name=smart-update-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-0 480
+ local pod=smart-update-pxc-0
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-0 condition met
smart-update-pxc-0.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-1 480
+ local pod=smart-update-pxc-1
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-1 condition met
smart-update-pxc-1.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-2 480
+ local pod=smart-update-pxc-2
+ local max_retry=480
+ local ns=
++ egrep '^(pxc|proxysql)$'
++ echo smart-update-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-2 condition met
smart-update-pxc-2.Ok
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-proxysql -uroot -proot_password -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h smart-update-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jCIeDi9SL5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZkiwpzMH3m
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.jCIeDi9SL5
++ cat /tmp/tmp.ZkiwpzMH3m
++ rm /tmp/tmp.jCIeDi9SL5 /tmp/tmp.ZkiwpzMH3m
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-proxysql -uroot -proot_password -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h smart-update-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0hNVPS12kX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0uVc1r3Bwm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0hNVPS12kX
++ cat /tmp/tmp.0uVc1r3Bwm
++ rm /tmp/tmp.0hNVPS12kX /tmp/tmp.0uVc1r3Bwm
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XMdhfr7PWb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7EyWER2yOr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XMdhfr7PWb
++ cat /tmp/tmp.7EyWER2yOr
++ rm /tmp/tmp.XMdhfr7PWb /tmp/tmp.7EyWER2yOr
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4JVmDQbCpS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cDCu1pcNxX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4JVmDQbCpS
++ cat /tmp/tmp.cDCu1pcNxX
++ rm /tmp/tmp.4JVmDQbCpS /tmp/tmp.cDCu1pcNxX
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-65c795cbdf-c9bcd
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ehFX0K8ANE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HN3O3ByTgl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ehFX0K8ANE
++ cat /tmp/tmp.HN3O3ByTgl
++ rm /tmp/tmp.ehFX0K8ANE /tmp/tmp.HN3O3ByTgl
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
++ is_keyring_plugin_in_use smart-update
++ local cluster=smart-update
++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NY1sJmWpfy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZgrAdlM3dy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NY1sJmWpfy
++ cat /tmp/tmp.ZgrAdlM3dy
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.NY1sJmWpfy /tmp/tmp.ZgrAdlM3dy
++ return 0
+ '[' '' ']'
+++ get_proxy smart-update
+++ local target_cluster=smart-update
++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.Oy3Zm9u18F
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.C1eIbgeOhG
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.Oy3Zm9u18F
++++ cat /tmp/tmp.C1eIbgeOhG
++++ rm /tmp/tmp.Oy3Zm9u18F /tmp/tmp.C1eIbgeOhG
++++ return 0
+++ [[ '' == \t\r\u\e ]]
++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.yYaeuVk39U
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.CoU0Il1lU5
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.yYaeuVk39U
++++ cat /tmp/tmp.CoU0Il1lU5
++++ rm /tmp/tmp.yYaeuVk39U /tmp/tmp.CoU0Il1lU5
++++ return 0
+++ [[ true == \t\r\u\e ]]
+++ echo smart-update-proxysql
+++ return
++ get_proxy_primary '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0
++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password'
++ local pod=smart-update-proxysql-0
+++ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 proxysql
+++ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';'
+++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password'
+++ local pod=smart-update-proxysql-0
+++ local container_name=proxysql
+++ set +o xtrace
++ local ip=smart-update-pxc-0.smart-update-pxc.smart-update1-13135.svc.cluster.local
+++ echo smart-update-pxc-0.smart-update-pxc.smart-update1-13135.svc.cluster.local
+++ wc -l
++ '[' 1 '!=' 1 ']'
++ echo smart-update-pxc-0.smart-update-pxc.smart-update1-13135.svc.cluster.local
++ cut -d. -f1
+ initial_primary=smart-update-pxc-0
+ kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.paByOjcu7v
++ mktemp
+ local LAST_ERR=/tmp/tmp.rgkGKxUwFh
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.paByOjcu7v
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ cat /tmp/tmp.rgkGKxUwFh
+ rm /tmp/tmp.paByOjcu7v /tmp/tmp.rgkGKxUwFh
+ return 0
+ sleep 7
+ desc 'check last pod to update'
+ set +o xtrace


-----------------------------------------------------------------------------------
check last pod to update
-----------------------------------------------------------------------------------

+ check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ local cluster=smart-update
+ local initial_primary=smart-update-pxc-0
+ local pxc_size=3
+ local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ set +x
Waiting for the last pod to update.........................................................................
smart-update-pxc-0 is REALLY the last one to update
+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JiKDvs7AlN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fK0X1QNdAM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.JiKDvs7AlN
++ cat /tmp/tmp.fK0X1QNdAM
++ rm /tmp/tmp.JiKDvs7AlN /tmp/tmp.fK0X1QNdAM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KC0GDeMpa0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.e9JWkn9iyq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KC0GDeMpa0
++ cat /tmp/tmp.e9JWkn9iyq
++ rm /tmp/tmp.KC0GDeMpa0 /tmp/tmp.e9JWkn9iyq
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zrNMdzLOca
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PJEsntOrxS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zrNMdzLOca
++ cat /tmp/tmp.PJEsntOrxS
++ rm /tmp/tmp.zrNMdzLOca /tmp/tmp.PJEsntOrxS
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CgnSuwqb1k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AX6520UQcN
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.CgnSuwqb1k
++ cat /tmp/tmp.AX6520UQcN
++ rm /tmp/tmp.CgnSuwqb1k /tmp/tmp.AX6520UQcN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wQ6tEenEJG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ysjctM3YK0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wQ6tEenEJG
++ cat /tmp/tmp.ysjctM3YK0
++ rm /tmp/tmp.wQ6tEenEJG /tmp/tmp.ysjctM3YK0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xluYLM3REx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bBg0bUpaHG
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xluYLM3REx
++ cat /tmp/tmp.bBg0bUpaHG
++ rm /tmp/tmp.xluYLM3REx /tmp/tmp.bBg0bUpaHG
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Z1GcQS2RQ7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gwrqkAHJKc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Z1GcQS2RQ7
++ cat /tmp/tmp.gwrqkAHJKc
++ rm /tmp/tmp.Z1GcQS2RQ7 /tmp/tmp.gwrqkAHJKc
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 6 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.f4F6juxOl6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.P9HllTr919
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.f4F6juxOl6
++ cat /tmp/tmp.P9HllTr919
++ rm /tmp/tmp.f4F6juxOl6 /tmp/tmp.P9HllTr919
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 7 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3OSTSkk0Pq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sfJLG7ldao
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3OSTSkk0Pq
++ cat /tmp/tmp.sfJLG7ldao
++ rm /tmp/tmp.3OSTSkk0Pq /tmp/tmp.sfJLG7ldao
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 8 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7jODthSvhE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NzMjsyOjEn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7jODthSvhE
++ cat /tmp/tmp.NzMjsyOjEn
++ rm /tmp/tmp.7jODthSvhE /tmp/tmp.NzMjsyOjEn
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 9 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rcns39pajI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K4pCEfL0GP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rcns39pajI
++ cat /tmp/tmp.K4pCEfL0GP
++ rm /tmp/tmp.rcns39pajI /tmp/tmp.K4pCEfL0GP
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 10 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pU8nxFnMKQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FJz1ObwrWi
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pU8nxFnMKQ
++ cat /tmp/tmp.FJz1ObwrWi
++ rm /tmp/tmp.pU8nxFnMKQ /tmp/tmp.FJz1ObwrWi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 11 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3weCazIBKg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CH9kbN0Awk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3weCazIBKg
++ cat /tmp/tmp.CH9kbN0Awk
++ rm /tmp/tmp.3weCazIBKg /tmp/tmp.CH9kbN0Awk
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qI2P94VrLE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HxecaMSBuX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.qI2P94VrLE
++ cat /tmp/tmp.HxecaMSBuX
++ rm /tmp/tmp.qI2P94VrLE /tmp/tmp.HxecaMSBuX
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.tV9BC7k0aM
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.ux96ll2ciH
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.tV9BC7k0aM
+++++ cat /tmp/tmp.ux96ll2ciH
+++++ rm /tmp/tmp.tV9BC7k0aM /tmp/tmp.ux96ll2ciH
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.cchbUSxwiC
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.yMRCkkPWHj
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.cchbUSxwiC
+++++ cat /tmp/tmp.yMRCkkPWHj
+++++ rm /tmp/tmp.cchbUSxwiC /tmp/tmp.yMRCkkPWHj
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-proxysql
++++ return
+++ local cluster_proxy=smart-update-proxysql
+++ echo proxysql
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TMCETN0AAa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8sTkGw8YKK
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TMCETN0AAa
++ cat /tmp/tmp.8sTkGw8YKK
++ rm /tmp/tmp.TMCETN0AAa /tmp/tmp.8sTkGw8YKK
++ return 0
+ [[ 2 == \2 ]]
++ seq 0 2
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eAuXGDVNb3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wMYNqytcGh
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eAuXGDVNb3
++ cat /tmp/tmp.wMYNqytcGh
++ rm /tmp/tmp.eAuXGDVNb3 /tmp/tmp.wMYNqytcGh
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ub8NUxglAE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ne7QG4ynl0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ub8NUxglAE
++ cat /tmp/tmp.ne7QG4ynl0
++ rm /tmp/tmp.ub8NUxglAE /tmp/tmp.ne7QG4ynl0
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4bHETenC6K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.L7xFASL1C5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4bHETenC6K
++ cat /tmp/tmp.L7xFASL1C5
++ rm /tmp/tmp.4bHETenC6K /tmp/tmp.L7xFASL1C5
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/conf/smart-update.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.kGBFSOkMxY
++ mktemp
+ local LAST_ERR=/tmp/tmp.4ydM2tTHyy
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/conf/smart-update.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.kGBFSOkMxY
perconaxtradbcluster.pxc.percona.com "smart-update" deleted
+ cat /tmp/tmp.4ydM2tTHyy
+ rm /tmp/tmp.kGBFSOkMxY /tmp/tmp.4ydM2tTHyy
+ return 0
+ kubectl_bin delete pvc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.sebV8I2eP7
++ mktemp
+ local LAST_ERR=/tmp/tmp.XTwg1aGvSn
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pvc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.sebV8I2eP7
persistentvolumeclaim "datadir-smart-update-pxc-0" deleted
persistentvolumeclaim "datadir-smart-update-pxc-1" deleted
persistentvolumeclaim "datadir-smart-update-pxc-2" deleted
persistentvolumeclaim "proxydata-smart-update-proxysql-0" deleted
persistentvolumeclaim "proxydata-smart-update-proxysql-1" deleted
+ cat /tmp/tmp.XTwg1aGvSn
+ rm /tmp/tmp.sebV8I2eP7 /tmp/tmp.XTwg1aGvSn
+ return 0
+ desc 'Updating HAProxy PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Updating HAProxy PXC cluster
-----------------------------------------------------------------------------------

+ cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/conf/smart-update-haproxy.yml /tmp/tmp.c7lkEUIUkH/smart-update-haproxy.yml
+ yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1785-7e3ddd30"' /tmp/tmp.c7lkEUIUkH/smart-update-haproxy.yml
+ spinup_pxc smart-update /tmp/tmp.c7lkEUIUkH/smart-update-haproxy.yml
+ local cluster=smart-update
+ local config=/tmp/tmp.c7lkEUIUkH/smart-update-haproxy.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.qnl95adFoq
++ mktemp
+ local LAST_ERR=/tmp/tmp.wWewXMaidm
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.qnl95adFoq
secret/my-cluster-secrets unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.wWewXMaidm
+ rm /tmp/tmp.qnl95adFoq /tmp/tmp.wWewXMaidm
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1785-7e3ddd30#'
+ local LAST_OUT=/tmp/tmp.M8OD4y5h9U
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-13135~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ local LAST_ERR=/tmp/tmp.goqjqe1Tof
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.M8OD4y5h9U
deployment.apps/pxc-client unchanged
+ cat /tmp/tmp.goqjqe1Tof
+ rm /tmp/tmp.M8OD4y5h9U /tmp/tmp.goqjqe1Tof
+ return 0
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ apply_config /tmp/tmp.c7lkEUIUkH/smart-update-haproxy.yml
+ '[' -z '' ']'
+ cat_config /tmp/tmp.c7lkEUIUkH/smart-update-haproxy.yml
+ kubectl_bin apply -f -
+ cat /tmp/tmp.c7lkEUIUkH/smart-update-haproxy.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1785-7e3ddd30#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-13135~
+ local LAST_OUT=/tmp/tmp.Il2lHWCUqt
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ local LAST_ERR=/tmp/tmp.9U1gaxsGLL
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Il2lHWCUqt
perconaxtradbcluster.pxc.percona.com/smart-update created
+ cat /tmp/tmp.9U1gaxsGLL
+ rm /tmp/tmp.Il2lHWCUqt /tmp/tmp.9U1gaxsGLL
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.YgL4hmBEt2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VUVJkSUPiE
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.YgL4hmBEt2
+++ cat /tmp/tmp.VUVJkSUPiE
+++ rm /tmp/tmp.YgL4hmBEt2 /tmp/tmp.VUVJkSUPiE
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ local proxy=smart-update-haproxy
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-13135
++ mktemp
+ local LAST_OUT=/tmp/tmp.v7rYwjBQm5
++ mktemp
+ local LAST_ERR=/tmp/tmp.QDHcYQ4uy7
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-13135
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-13135
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-13135
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.v7rYwjBQm5
+ cat /tmp/tmp.QDHcYQ4uy7
error: no matching resources found
+ rm /tmp/tmp.v7rYwjBQm5 /tmp/tmp.QDHcYQ4uy7
+ return 1
+ true
+ wait_for_running smart-update-haproxy 1
+ local name=smart-update-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-haproxy-0 480
+ local pod=smart-update-haproxy-0
+ local max_retry=480
+ local ns=
++ echo smart-update-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/smart-update-haproxy-0 condition met
smart-update-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init), haproxy-init (init)
.Ok
+ wait_for_running smart-update-pxc 3
+ local name=smart-update-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-0 480
+ local pod=smart-update-pxc-0
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-0
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-0 condition met
smart-update-pxc-0.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-1 480
+ local pod=smart-update-pxc-1
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-1 condition met
smart-update-pxc-1.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-2 480
+ local pod=smart-update-pxc-2
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-2 condition met
smart-update-pxc-2.Ok
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IY599N3xDr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iaX0ZewKRx
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IY599N3xDr
++ cat /tmp/tmp.iaX0ZewKRx
++ rm /tmp/tmp.IY599N3xDr /tmp/tmp.iaX0ZewKRx
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MQJzBL0jwS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TvjNg1j1VT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.MQJzBL0jwS
++ cat /tmp/tmp.TvjNg1j1VT
++ rm /tmp/tmp.MQJzBL0jwS /tmp/tmp.TvjNg1j1VT
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EcUaWF2vc2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.phQDc3UfOe
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.EcUaWF2vc2
++ cat /tmp/tmp.phQDc3UfOe
++ rm /tmp/tmp.EcUaWF2vc2 /tmp/tmp.phQDc3UfOe
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.64Wr7o6rwu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.78ilIlngTl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.64Wr7o6rwu
++ cat /tmp/tmp.78ilIlngTl
++ rm /tmp/tmp.64Wr7o6rwu /tmp/tmp.78ilIlngTl
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oUXnBIHZlE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.La1O1ItWnr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.oUXnBIHZlE
++ cat /tmp/tmp.La1O1ItWnr
++ rm /tmp/tmp.oUXnBIHZlE /tmp/tmp.La1O1ItWnr
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
++ is_keyring_plugin_in_use smart-update
++ local cluster=smart-update
++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nIRTL0IAs2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9P5YBCXN7m
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nIRTL0IAs2
++ cat /tmp/tmp.9P5YBCXN7m
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.nIRTL0IAs2 /tmp/tmp.9P5YBCXN7m
++ return 0
+ '[' '' ']'
++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password'
++ local 'command=SELECT @@hostname hostname;'
++ local 'uri=-h smart-update-haproxy -uroot -proot_password'
+++ get_client_pod
+++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.fKK0YE3RUW
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.BVoO1oDM9n
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.fKK0YE3RUW
+++ cat /tmp/tmp.BVoO1oDM9n
+++ rm /tmp/tmp.fKK0YE3RUW /tmp/tmp.BVoO1oDM9n
+++ return 0
++ client_pod=pxc-client-65c795cbdf-c9bcd
++ wait_pod pxc-client-65c795cbdf-c9bcd
++ local pod=pxc-client-65c795cbdf-c9bcd
++ local max_retry=480
++ local ns=
+++ echo pxc-client-65c795cbdf-c9bcd
+++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+++ egrep '^(pxc|proxysql)$'
++ local container=
++ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
++ set +o xtrace
+ initial_primary=smart-update-pxc-0
+ kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.oJUBmwqfM2
++ mktemp
+ local LAST_ERR=/tmp/tmp.XaOdWCCVtf
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.oJUBmwqfM2
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ cat /tmp/tmp.XaOdWCCVtf
+ rm /tmp/tmp.oJUBmwqfM2 /tmp/tmp.XaOdWCCVtf
+ return 0
+ sleep 7
+ desc 'check last pod to update'
+ set +o xtrace


-----------------------------------------------------------------------------------
check last pod to update
-----------------------------------------------------------------------------------

+ check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ local cluster=smart-update
+ local initial_primary=smart-update-pxc-0
+ local pxc_size=3
+ local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ set +x
Waiting for the last pod to update...........................................
smart-update-pxc-0 is REALLY the last one to update
+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FeRQW6mCmy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aEuPGTDtjE
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FeRQW6mCmy
++ cat /tmp/tmp.aEuPGTDtjE
++ rm /tmp/tmp.FeRQW6mCmy /tmp/tmp.aEuPGTDtjE
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.biN6k1cONp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GwC4mDiTY5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.biN6k1cONp
++ cat /tmp/tmp.GwC4mDiTY5
++ rm /tmp/tmp.biN6k1cONp /tmp/tmp.GwC4mDiTY5
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GHwoFvXihp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CsnwbaKH7M
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GHwoFvXihp
++ cat /tmp/tmp.CsnwbaKH7M
++ rm /tmp/tmp.GHwoFvXihp /tmp/tmp.CsnwbaKH7M
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2P1sCPSy0L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2yP057aXGs
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2P1sCPSy0L
++ cat /tmp/tmp.2yP057aXGs
++ rm /tmp/tmp.2P1sCPSy0L /tmp/tmp.2yP057aXGs
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.k9aYOMF6qN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G7b4ka1R3k
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.k9aYOMF6qN
++ cat /tmp/tmp.G7b4ka1R3k
++ rm /tmp/tmp.k9aYOMF6qN /tmp/tmp.G7b4ka1R3k
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Qzr7hPptt2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4ow55zPMld
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Qzr7hPptt2
++ cat /tmp/tmp.4ow55zPMld
++ rm /tmp/tmp.Qzr7hPptt2 /tmp/tmp.4ow55zPMld
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WG9Fo7hZJk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tp9uE5ig5Y
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WG9Fo7hZJk
++ cat /tmp/tmp.tp9uE5ig5Y
++ rm /tmp/tmp.WG9Fo7hZJk /tmp/tmp.tp9uE5ig5Y
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 6 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gT5ljXcoZC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4lOLY9k1KZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gT5ljXcoZC
++ cat /tmp/tmp.4lOLY9k1KZ
++ rm /tmp/tmp.gT5ljXcoZC /tmp/tmp.4lOLY9k1KZ
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DoIClqmeez
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B3Bcnuqq0u
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DoIClqmeez
++ cat /tmp/tmp.B3Bcnuqq0u
++ rm /tmp/tmp.DoIClqmeez /tmp/tmp.B3Bcnuqq0u
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.6EeCAEDEwW
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.nvpgLHJtCB
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.6EeCAEDEwW
+++++ cat /tmp/tmp.nvpgLHJtCB
+++++ rm /tmp/tmp.6EeCAEDEwW /tmp/tmp.nvpgLHJtCB
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L1rxofvbbl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xvcCfOzBnZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.L1rxofvbbl
++ cat /tmp/tmp.xvcCfOzBnZ
++ rm /tmp/tmp.L1rxofvbbl /tmp/tmp.xvcCfOzBnZ
++ return 0
+ [[ 2 == \2 ]]
++ seq 0 2
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zyztfdsqdQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.X7j6bX9VU4
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zyztfdsqdQ
++ cat /tmp/tmp.X7j6bX9VU4
++ rm /tmp/tmp.zyztfdsqdQ /tmp/tmp.X7j6bX9VU4
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KkVCdslA31
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3813xVM6VT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KkVCdslA31
++ cat /tmp/tmp.3813xVM6VT
++ rm /tmp/tmp.KkVCdslA31 /tmp/tmp.3813xVM6VT
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hs5ghYYn3X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1oStBMEvpY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hs5ghYYn3X
++ cat /tmp/tmp.1oStBMEvpY
++ rm /tmp/tmp.hs5ghYYn3X /tmp/tmp.1oStBMEvpY
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ kubectl_bin delete -f /tmp/tmp.c7lkEUIUkH/smart-update-haproxy.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.8nN1ys9Lzj
++ mktemp
+ local LAST_ERR=/tmp/tmp.fEYVRDWhEk
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /tmp/tmp.c7lkEUIUkH/smart-update-haproxy.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.8nN1ys9Lzj
perconaxtradbcluster.pxc.percona.com "smart-update" deleted
+ cat /tmp/tmp.fEYVRDWhEk
+ rm /tmp/tmp.8nN1ys9Lzj /tmp/tmp.fEYVRDWhEk
+ return 0
+ kubectl_bin delete pvc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.t1RzIAVeTM
++ mktemp
+ local LAST_ERR=/tmp/tmp.NZWCXtGYHY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pvc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.t1RzIAVeTM
persistentvolumeclaim "datadir-smart-update-pxc-0" deleted
persistentvolumeclaim "datadir-smart-update-pxc-1" deleted
persistentvolumeclaim "datadir-smart-update-pxc-2" deleted
+ cat /tmp/tmp.NZWCXtGYHY
+ rm /tmp/tmp.t1RzIAVeTM /tmp/tmp.NZWCXtGYHY
+ return 0
+ desc 'Updating PXC cluster with version service available but disabled'
+ set +o xtrace


-----------------------------------------------------------------------------------
Updating PXC cluster with version service available but disabled
-----------------------------------------------------------------------------------

+ cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/conf/smart-update-version-service-reachable.yml /tmp/tmp.c7lkEUIUkH/smart-update-version-service-reachable.yml
+ yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1785-7e3ddd30"' /tmp/tmp.c7lkEUIUkH/smart-update-version-service-reachable.yml
+ spinup_pxc smart-update /tmp/tmp.c7lkEUIUkH/smart-update-version-service-reachable.yml
+ local cluster=smart-update
+ local config=/tmp/tmp.c7lkEUIUkH/smart-update-version-service-reachable.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.lGa2SDe6iZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.e1kcJ7GwST
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lGa2SDe6iZ
secret/my-cluster-secrets unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.e1kcJ7GwST
+ rm /tmp/tmp.lGa2SDe6iZ /tmp/tmp.e1kcJ7GwST
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1785-7e3ddd30#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ local LAST_OUT=/tmp/tmp.8tEq3nsfzD
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-13135~
+ local LAST_ERR=/tmp/tmp.0ocNmxicxw
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.8tEq3nsfzD
deployment.apps/pxc-client unchanged
+ cat /tmp/tmp.0ocNmxicxw
+ rm /tmp/tmp.8tEq3nsfzD /tmp/tmp.0ocNmxicxw
+ return 0
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ apply_config /tmp/tmp.c7lkEUIUkH/smart-update-version-service-reachable.yml
+ '[' -z '' ']'
+ cat_config /tmp/tmp.c7lkEUIUkH/smart-update-version-service-reachable.yml
+ kubectl_bin apply -f -
+ cat /tmp/tmp.c7lkEUIUkH/smart-update-version-service-reachable.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ local LAST_OUT=/tmp/tmp.vXkwcwlDfz
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1785-7e3ddd30#'
++ mktemp
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-13135~
+ local LAST_ERR=/tmp/tmp.X3QUAL2x7L
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vXkwcwlDfz
perconaxtradbcluster.pxc.percona.com/smart-update created
+ cat /tmp/tmp.X3QUAL2x7L
+ rm /tmp/tmp.vXkwcwlDfz /tmp/tmp.X3QUAL2x7L
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.rtPFUZPg9a
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NfkLUPCQpR
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.rtPFUZPg9a
+++ cat /tmp/tmp.NfkLUPCQpR
+++ rm /tmp/tmp.rtPFUZPg9a /tmp/tmp.NfkLUPCQpR
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ local proxy=smart-update-haproxy
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-13135
++ mktemp
+ local LAST_OUT=/tmp/tmp.EjrK8w5NQ5
++ mktemp
+ local LAST_ERR=/tmp/tmp.AthjfwMBZO
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-13135
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-13135
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-13135
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.EjrK8w5NQ5
+ cat /tmp/tmp.AthjfwMBZO
error: no matching resources found
+ rm /tmp/tmp.EjrK8w5NQ5 /tmp/tmp.AthjfwMBZO
+ return 1
+ true
+ wait_for_running smart-update-haproxy 1
+ local name=smart-update-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-haproxy-0 480
+ local pod=smart-update-haproxy-0
+ local max_retry=480
+ local ns=
++ echo smart-update-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/smart-update-haproxy-0 condition met
smart-update-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init), haproxy-init (init)
.Ok
+ wait_for_running smart-update-pxc 3
+ local name=smart-update-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-0 480
+ local pod=smart-update-pxc-0
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-0 condition met
smart-update-pxc-0.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-1 480
+ local pod=smart-update-pxc-1
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-1 condition met
smart-update-pxc-1.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-2 480
+ local pod=smart-update-pxc-2
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-2 condition met
smart-update-pxc-2.Ok
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PGL7ViSTQm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PnYmAZNZ7k
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PGL7ViSTQm
++ cat /tmp/tmp.PnYmAZNZ7k
++ rm /tmp/tmp.PGL7ViSTQm /tmp/tmp.PnYmAZNZ7k
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.g67TcVkVis
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rw5y09YFqd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.g67TcVkVis
++ cat /tmp/tmp.rw5y09YFqd
++ rm /tmp/tmp.g67TcVkVis /tmp/tmp.rw5y09YFqd
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VgkZlLaXMq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6w9jwVKcPX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VgkZlLaXMq
++ cat /tmp/tmp.6w9jwVKcPX
++ rm /tmp/tmp.VgkZlLaXMq /tmp/tmp.6w9jwVKcPX
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.itBg9a4xzf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.R2gQOdlM2Z
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.itBg9a4xzf
++ cat /tmp/tmp.R2gQOdlM2Z
++ rm /tmp/tmp.itBg9a4xzf /tmp/tmp.R2gQOdlM2Z
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6ZQDGrcVvj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0KTLmLObMG
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6ZQDGrcVvj
++ cat /tmp/tmp.0KTLmLObMG
++ rm /tmp/tmp.6ZQDGrcVvj /tmp/tmp.0KTLmLObMG
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
++ is_keyring_plugin_in_use smart-update
++ local cluster=smart-update
++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5g96ReZrXp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b2ugbf8Qcv
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.5g96ReZrXp
++ cat /tmp/tmp.b2ugbf8Qcv
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.5g96ReZrXp /tmp/tmp.b2ugbf8Qcv
++ return 0
+ '[' '' ']'
++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password'
++ local 'command=SELECT @@hostname hostname;'
++ local 'uri=-h smart-update-haproxy -uroot -proot_password'
+++ get_client_pod
+++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.M5JQXkDnao
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.oe5GK4bKlu
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.M5JQXkDnao
+++ cat /tmp/tmp.oe5GK4bKlu
+++ rm /tmp/tmp.M5JQXkDnao /tmp/tmp.oe5GK4bKlu
+++ return 0
++ client_pod=pxc-client-65c795cbdf-c9bcd
++ wait_pod pxc-client-65c795cbdf-c9bcd
++ local pod=pxc-client-65c795cbdf-c9bcd
++ local max_retry=480
++ local ns=
+++ echo pxc-client-65c795cbdf-c9bcd
+++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+++ egrep '^(pxc|proxysql)$'
++ local container=
++ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
++ set +o xtrace
+ initial_primary=smart-update-pxc-0
+ kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.nTmKiYpbBc
++ mktemp
+ local LAST_ERR=/tmp/tmp.51EhHKYYCS
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nTmKiYpbBc
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ cat /tmp/tmp.51EhHKYYCS
+ rm /tmp/tmp.nTmKiYpbBc /tmp/tmp.51EhHKYYCS
+ return 0
+ sleep 7
+ desc 'check last pod to update'
+ set +o xtrace


-----------------------------------------------------------------------------------
check last pod to update
-----------------------------------------------------------------------------------

+ check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ local cluster=smart-update
+ local initial_primary=smart-update-pxc-0
+ local pxc_size=3
+ local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ set +x
Waiting for the last pod to update...........................................
smart-update-pxc-0 is REALLY the last one to update
+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IkxQnk2jdr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sdJPERtjLM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IkxQnk2jdr
++ cat /tmp/tmp.sdJPERtjLM
++ rm /tmp/tmp.IkxQnk2jdr /tmp/tmp.sdJPERtjLM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BopBTLACiT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bOXAQsXjlW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BopBTLACiT
++ cat /tmp/tmp.bOXAQsXjlW
++ rm /tmp/tmp.BopBTLACiT /tmp/tmp.bOXAQsXjlW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.51KbL12p1P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FsPakYAaot
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.51KbL12p1P
++ cat /tmp/tmp.FsPakYAaot
++ rm /tmp/tmp.51KbL12p1P /tmp/tmp.FsPakYAaot
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LDhZyqcAyv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.39SvbZrg0n
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LDhZyqcAyv
++ cat /tmp/tmp.39SvbZrg0n
++ rm /tmp/tmp.LDhZyqcAyv /tmp/tmp.39SvbZrg0n
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Uu59jDi1vl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zHZQ94027M
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Uu59jDi1vl
++ cat /tmp/tmp.zHZQ94027M
++ rm /tmp/tmp.Uu59jDi1vl /tmp/tmp.zHZQ94027M
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.APamYFLQq9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Bxc2XaLxeI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.APamYFLQq9
++ cat /tmp/tmp.Bxc2XaLxeI
++ rm /tmp/tmp.APamYFLQq9 /tmp/tmp.Bxc2XaLxeI
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.htDeTvCTtB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QB6h3vgsj7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.htDeTvCTtB
++ cat /tmp/tmp.QB6h3vgsj7
++ rm /tmp/tmp.htDeTvCTtB /tmp/tmp.QB6h3vgsj7
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 6 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xQvZ0HQzDQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HV3bqLFzIj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xQvZ0HQzDQ
++ cat /tmp/tmp.HV3bqLFzIj
++ rm /tmp/tmp.xQvZ0HQzDQ /tmp/tmp.HV3bqLFzIj
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 7 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c3CajgHb9w
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ieLs9nS3TO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.c3CajgHb9w
++ cat /tmp/tmp.ieLs9nS3TO
++ rm /tmp/tmp.c3CajgHb9w /tmp/tmp.ieLs9nS3TO
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DSIGTm8WPY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AlfoM4pRBk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DSIGTm8WPY
++ cat /tmp/tmp.AlfoM4pRBk
++ rm /tmp/tmp.DSIGTm8WPY /tmp/tmp.AlfoM4pRBk
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.J9Xu7iSL4j
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.u66MyuI2f7
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.J9Xu7iSL4j
+++++ cat /tmp/tmp.u66MyuI2f7
+++++ rm /tmp/tmp.J9Xu7iSL4j /tmp/tmp.u66MyuI2f7
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZRhdL2dNL5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BgBsrQ3osy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZRhdL2dNL5
++ cat /tmp/tmp.BgBsrQ3osy
++ rm /tmp/tmp.ZRhdL2dNL5 /tmp/tmp.BgBsrQ3osy
++ return 0
+ [[ 2 == \2 ]]
++ seq 0 2
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nHueOT2fuw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gU4I9VIjf4
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nHueOT2fuw
++ cat /tmp/tmp.gU4I9VIjf4
++ rm /tmp/tmp.nHueOT2fuw /tmp/tmp.gU4I9VIjf4
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VRJKYmZZit
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xNZaY96gLQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VRJKYmZZit
++ cat /tmp/tmp.xNZaY96gLQ
++ rm /tmp/tmp.VRJKYmZZit /tmp/tmp.xNZaY96gLQ
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aLH7D5kInn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xGxjSKDIcp
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.aLH7D5kInn
++ cat /tmp/tmp.xGxjSKDIcp
++ rm /tmp/tmp.aLH7D5kInn /tmp/tmp.xGxjSKDIcp
++ return 0
+ client_pod=pxc-client-65c795cbdf-c9bcd
+ wait_pod pxc-client-65c795cbdf-c9bcd
+ local pod=pxc-client-65c795cbdf-c9bcd
+ local max_retry=480
+ local ns=
++ echo pxc-client-65c795cbdf-c9bcd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65c795cbdf-c9bcd condition met
pxc-client-65c795cbdf-c9bcd.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.c7lkEUIUkH/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.c7lkEUIUkH/select-1.sql
+ kubectl_bin delete -f /tmp/tmp.c7lkEUIUkH/smart-update-version-service-reachable.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZXQ9362JNW
++ mktemp
+ local LAST_ERR=/tmp/tmp.Pet8I2OCuG
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /tmp/tmp.c7lkEUIUkH/smart-update-version-service-reachable.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ZXQ9362JNW
perconaxtradbcluster.pxc.percona.com "smart-update" deleted
+ cat /tmp/tmp.Pet8I2OCuG
+ rm /tmp/tmp.ZXQ9362JNW /tmp/tmp.Pet8I2OCuG
+ return 0
+ kubectl_bin delete pvc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.HsglXyv6Nj
++ mktemp
+ local LAST_ERR=/tmp/tmp.GFrHcvxIWR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pvc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.HsglXyv6Nj
persistentvolumeclaim "datadir-smart-update-pxc-0" deleted
persistentvolumeclaim "datadir-smart-update-pxc-1" deleted
persistentvolumeclaim "datadir-smart-update-pxc-2" deleted
+ cat /tmp/tmp.GFrHcvxIWR
+ rm /tmp/tmp.HsglXyv6Nj /tmp/tmp.GFrHcvxIWR
+ return 0
+ desc cleanup
+ set +o xtrace


-----------------------------------------------------------------------------------
cleanup
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/conf/vs.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.131XgfLXdF
++ mktemp
+ local LAST_ERR=/tmp/tmp.9mimNxe4Hy
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1785/e2e-tests/smart-update1/conf/vs.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.131XgfLXdF
deployment.apps "version-service" deleted
service "version-service" deleted
+ cat /tmp/tmp.9mimNxe4Hy
+ rm /tmp/tmp.131XgfLXdF /tmp/tmp.9mimNxe4Hy
+ return 0
+ destroy smart-update1-13135
+ local namespace=smart-update1-13135
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
++ get_operator_pod
+ grep -v 'the object has been modified'
++ local label_prefix=app.kubernetes.io/
+ grep -v 'get backup status: Job.batch'
+ tee /tmp/tmp.c7lkEUIUkH/operator.log
+++ grep -c percona-xtradb-cluster-operator
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ sort -u
+ grep -v level=info
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tbYLGxl4QG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AyaxgkpuhS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.tbYLGxl4QG
++ cat /tmp/tmp.AyaxgkpuhS
++ rm /tmp/tmp.tbYLGxl4QG /tmp/tmp.AyaxgkpuhS
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-66cc8ffb95-nb2nj
++ mktemp
+ local LAST_OUT=/tmp/tmp.rtzYKZztkU
++ mktemp
+ local LAST_ERR=/tmp/tmp.Haokg4Y9ya
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-66cc8ffb95-nb2nj
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.rtzYKZztkU
+ cat /tmp/tmp.Haokg4Y9ya
+ rm /tmp/tmp.rtzYKZztkU /tmp/tmp.Haokg4Y9ya
+ return 0
2024-08-09T11:35:13.201Z	INFO	setup	Manager starting up	{"gitCommit": "7e3ddd30045a1f5403cbb31a3db0e57c5edb140b", "gitBranch": "PR-1785-7e3ddd30", "buildTime": "2024-08-09T09:47:37Z", "goVersion": "go1.22.6", "os": "linux", "arch": "amd64"}
2024-08-09T11:35:13.201Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.27.16-gke.1051000"}
2024-08-09T11:35:13.203Z	INFO	setup	Registering Components.
2024-08-09T11:35:14.982Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2024-08-09T11:35:14.986Z	INFO	controller-runtime.metrics	Starting metrics server
2024-08-09T11:35:14.986Z	INFO	setup	Starting the Cmd.
2024-08-09T11:35:14.987Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2024-08-09T11:35:14.987Z	INFO	controller-runtime.webhook	Starting webhook server
2024-08-09T11:35:14.987Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2024-08-09T11:35:15.070Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate
2024-08-09T11:35:15.070Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2024-08-09T11:35:15.071Z	INFO	controller-runtime.certwatcher	Starting certificate watcher
2024-08-09T11:35:15.088Z	INFO	attempting to acquire leader lease pxc-operator/08db1feb.percona.com...
2024-08-09T11:35:15.115Z	INFO	successfully acquired lease pxc-operator/08db1feb.percona.com
2024-08-09T11:35:15.116Z	DEBUG	events	percona-xtradb-cluster-operator-66cc8ffb95-nb2nj_48446db6-1fa1-47aa-8c89-3d0034c33a0c became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"457976c6-8b6a-49a9-a7f5-343dda3b14c2","apiVersion":"coordination.k8s.io/v1","resourceVersion":"60816"}, "reason": "LeaderElection"}
2024-08-09T11:35:15.116Z	INFO	Starting Controller	{"controller": "pxc-controller"}
2024-08-09T11:35:15.116Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"}
2024-08-09T11:35:15.117Z	INFO	Starting Controller	{"controller": "pxcbackup-controller"}
2024-08-09T11:35:15.117Z	INFO	Starting Controller	{"controller": "pxcrestore-controller"}
2024-08-09T11:35:15.117Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2024-08-09T11:35:15.117Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2024-08-09T11:35:15.228Z	INFO	Starting workers	{"controller": "pxc-controller", "worker count": 1}
2024-08-09T11:35:15.228Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "worker count": 1}
2024-08-09T11:35:15.236Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "worker count": 1}
2024-08-09T11:37:43.514Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "142dcfa6-20dd-4c4e-9abc-fb2a9b2c4321", "object": "smart-update-pxc"}
2024-08-09T11:37:43.589Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "142dcfa6-20dd-4c4e-9abc-fb2a9b2c4321", "object": "smart-update-pxc"}
2024-08-09T11:37:43.589Z	DEBUG	Waiting for ProxySQL to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "142dcfa6-20dd-4c4e-9abc-fb2a9b2c4321"}
2024-08-09T11:37:43.719Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "142dcfa6-20dd-4c4e-9abc-fb2a9b2c4321", "object": "smart-update-pxc-unready"}
2024-08-09T11:37:43.864Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "142dcfa6-20dd-4c4e-9abc-fb2a9b2c4321", "object": "smart-update-proxysql"}
2024-08-09T11:37:43.990Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "142dcfa6-20dd-4c4e-9abc-fb2a9b2c4321", "object": "smart-update-proxysql"}
2024-08-09T11:37:44.133Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "142dcfa6-20dd-4c4e-9abc-fb2a9b2c4321", "object": "smart-update-proxysql-unready"}
2024-08-09T11:37:44.555Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "b78881ff-b6c5-4564-810e-c35924914f81", "object": "smart-update-pxc"}
2024-08-09T11:37:44.608Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "b78881ff-b6c5-4564-810e-c35924914f81", "object": "smart-update-proxysql"}
2024-08-09T11:37:44.624Z	DEBUG	Waiting for ProxySQL to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "b78881ff-b6c5-4564-810e-c35924914f81"}
2024-08-09T11:37:49.566Z	DEBUG	Waiting for ProxySQL to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "cd5408f3-bb48-4647-95f3-8c86c2ebcd01"}
2024-08-09T11:37:54.669Z	DEBUG	Waiting for ProxySQL to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "ea240f55-2da3-436d-9d88-6d5c5c639913"}
2024-08-09T11:37:59.833Z	DEBUG	Waiting for ProxySQL to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "42b426f8-cde6-4708-88cb-3f953fe43b90"}
2024-08-09T11:38:00.159Z	DEBUG	Waiting for ProxySQL to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "83c536a8-5912-4001-b3fb-d6f8a9b79d66"}
2024-08-09T11:38:05.069Z	DEBUG	Waiting for ProxySQL to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "bb169cfb-54b9-4501-94fb-17f573c3d8f9"}
2024-08-09T11:38:10.139Z	DEBUG	Waiting for ProxySQL to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "03d2e095-9bb7-4c1d-b05f-6111a9be1c3b"}
2024-08-09T11:38:15.211Z	DEBUG	Waiting for ProxySQL to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "32fdc76c-c49a-4eb2-b3dd-b083319072fd"}
2024-08-09T11:38:20.282Z	DEBUG	Waiting for ProxySQL to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "ce4caef3-82b7-4b8d-8fcd-00e47e3af5d8"}
2024-08-09T11:38:57.071Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "83fe7879-9f9c-4c97-80ba-1bc3e000917a", "user": "operator"}
2024-08-09T11:38:57.114Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "83fe7879-9f9c-4c97-80ba-1bc3e000917a", "user": "monitor"}
2024-08-09T11:38:57.188Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "83fe7879-9f9c-4c97-80ba-1bc3e000917a"}
2024-08-09T11:38:57.224Z	INFO	monitor user privileges granted	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "83fe7879-9f9c-4c97-80ba-1bc3e000917a"}
2024-08-09T11:38:57.270Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "83fe7879-9f9c-4c97-80ba-1bc3e000917a", "user": "xtrabackup"}
2024-08-09T11:38:57.327Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "83fe7879-9f9c-4c97-80ba-1bc3e000917a"}
2024-08-09T11:38:57.368Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "83fe7879-9f9c-4c97-80ba-1bc3e000917a", "user": "replication"}
2024-08-09T11:38:58.402Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "83fe7879-9f9c-4c97-80ba-1bc3e000917a", "err": "get primary pxc pod: not found"}
2024-08-09T11:39:03.078Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "841a0158-19bc-4cf5-9b63-31b53f5d5114", "err": "get primary pxc pod: not found"}
2024-08-09T11:39:09.252Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "34422270-2a14-4446-add9-f123ca9a8812", "err": "get primary pxc pod: not found"}
2024-08-09T11:39:15.382Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "77b42586-7946-4efa-94b7-2f3f0bbfe072", "err": "get primary pxc pod: not found"}
2024-08-09T11:41:26.039Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "43b333f6-c9f8-44d1-bed9-fc295336c504", "user": "root"}
2024-08-09T11:41:27.174Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "43b333f6-c9f8-44d1-bed9-fc295336c504", "new version": "8.0.19-10"}
2024-08-09T11:41:28.549Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "43b333f6-c9f8-44d1-bed9-fc295336c504"}
2024-08-09T11:41:33.372Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d49b7ab6-90e1-4954-b572-fc5f3e7fb843"}
2024-08-09T11:41:39.621Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "0e541f3e-4881-47de-ba16-aab6b6d73a12"}
2024-08-09T11:41:45.950Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "04a1c1bd-404e-415f-ad4e-5291ecc2d078"}
2024-08-09T11:41:52.154Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d1a61525-0a9d-4858-b146-28ca7bb0bd80"}
2024-08-09T11:41:58.410Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c1cc01b3-ec20-45b0-893a-ae821107c608"}
2024-08-09T11:42:04.645Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "61d46a1f-8171-44d2-828b-1b2841c84feb"}
2024-08-09T11:42:10.914Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "7125261c-7410-4571-afaa-2108bc514593"}
2024-08-09T11:42:17.313Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "fe3fe4e4-3e25-4ff3-ac85-28b23db10ebd"}
2024-08-09T11:42:23.472Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4aa80309-8b61-46e8-95f8-c6a3a8d2bde4"}
2024-08-09T11:42:29.738Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "8c2396ce-0dc4-4788-9065-28f4fa6a5167"}
2024-08-09T11:42:36.770Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "776ce63e-89eb-43db-92fe-aab2062ac9b9"}
2024-08-09T11:42:36.871Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "object": "smart-update-pxc"}
2024-08-09T11:42:37.964Z	INFO	statefulSet was changed, run smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b"}
2024-08-09T11:42:37.973Z	INFO	apply changes to secondary pod	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-2"}
2024-08-09T11:42:37.973Z	INFO	primary pod	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-0.smart-update-pxc.smart-update1-13135.svc.cluster.local"}
2024-08-09T11:42:39.465Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b"}
2024-08-09T11:42:58.224Z	INFO	pod is waiting	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-2", "reason": "PodInitializing"}
2024-08-09T11:44:08.245Z	INFO	pod is running	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-2"}
2024-08-09T11:44:58.269Z	INFO	pod present in hostgroups	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-2"}
2024-08-09T11:44:58.279Z	INFO	apply changes to secondary pod	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-1"}
2024-08-09T11:44:58.279Z	INFO	pod is online	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-2"}
2024-08-09T11:45:18.551Z	INFO	pod is waiting	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-1", "reason": "PodInitializing"}
2024-08-09T11:46:18.581Z	INFO	pod is running	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-1"}
2024-08-09T11:47:08.605Z	INFO	pod present in hostgroups	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-1"}
2024-08-09T11:47:08.612Z	INFO	apply changes to primary pod	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-0"}
2024-08-09T11:47:08.612Z	INFO	pod is online	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-1"}
2024-08-09T11:47:28.894Z	INFO	pod is waiting	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-0", "reason": "PodInitializing"}
2024-08-09T11:48:38.911Z	INFO	pod is running	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-0"}
2024-08-09T11:49:28.936Z	INFO	pod present in hostgroups	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-0"}
2024-08-09T11:49:28.943Z	INFO	pod is online	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "pod name": "smart-update-pxc-0"}
2024-08-09T11:49:28.943Z	INFO	smart update finished	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b"}
2024-08-09T11:49:29.025Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c5339ba1-9aac-41ed-8214-a063d643073b", "new version": "8.0.36-28.1"}
2024-08-09T11:49:33.175Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "78a8f1e2-b3c3-44a0-a381-d544ab9c0bef"}
2024-08-09T11:49:37.039Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "52b7bb2e-959a-4c94-91b5-65daee7a04f9"}
2024-08-09T11:49:43.210Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "0624c2e7-51cd-428b-b09b-a680034345f9"}
2024-08-09T11:49:49.584Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "e3a3d967-d27d-4da9-b6dc-21e96b2c0ae4"}
2024-08-09T11:50:44.770Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "9f156287-10ca-4aeb-8927-2461d959086b", "object": "smart-update-pxc"}
2024-08-09T11:50:44.861Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "9f156287-10ca-4aeb-8927-2461d959086b", "object": "smart-update-pxc"}
2024-08-09T11:50:44.861Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "9f156287-10ca-4aeb-8927-2461d959086b"}
2024-08-09T11:50:44.991Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "9f156287-10ca-4aeb-8927-2461d959086b", "object": "smart-update-pxc-unready"}
2024-08-09T11:50:45.118Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "9f156287-10ca-4aeb-8927-2461d959086b", "object": "smart-update-haproxy"}
2024-08-09T11:50:45.264Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "9f156287-10ca-4aeb-8927-2461d959086b", "object": "smart-update-haproxy"}
2024-08-09T11:50:45.359Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "9f156287-10ca-4aeb-8927-2461d959086b", "object": "smart-update-haproxy"}
2024-08-09T11:50:45.518Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "9f156287-10ca-4aeb-8927-2461d959086b", "object": "smart-update-haproxy-replicas"}
2024-08-09T11:50:45.685Z	INFO	KubeAPIWarningLogger	.metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 9b6d4775-8c63-458c-bde8-9354c3dc2a6c
2024-08-09T11:50:45.787Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "e42bf956-efeb-4cc8-98d5-37d82ef85223", "object": "smart-update-pxc"}
2024-08-09T11:50:45.836Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "e42bf956-efeb-4cc8-98d5-37d82ef85223", "object": "smart-update-haproxy"}
2024-08-09T11:50:45.848Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "e42bf956-efeb-4cc8-98d5-37d82ef85223"}
2024-08-09T11:50:47.854Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "7a34066f-da81-4ef9-addc-997adb475458"}
2024-08-09T11:50:52.918Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "a0f1cec4-1826-409e-a45f-efd8a4d0522e"}
2024-08-09T11:50:57.988Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "2b702ed3-8308-4252-9783-a136f78edd2b"}
2024-08-09T11:51:03.075Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "a161668b-47c6-4e2f-ae3f-90409a1350e3"}
2024-08-09T11:51:08.152Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "44599810-a48f-4bc2-b51e-b3541ab9ae57"}
2024-08-09T11:51:13.245Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "10445dcf-7aa6-426c-a054-a73ad1d06acb"}
2024-08-09T11:51:18.313Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "5b2203ff-ef09-4526-aeb2-13a4550ecd03"}
2024-08-09T11:51:23.382Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "9d1c0d9e-8662-41a6-89ef-4ca8c601a376"}
2024-08-09T11:51:28.456Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "be1a3b3d-c933-4d24-aecc-2c09164947fa"}
2024-08-09T11:51:33.532Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "743df8b6-8608-40d4-afa7-8b0b1640638e"}
2024-08-09T11:51:38.605Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "98d90014-b964-4fb0-bdeb-aa07f7b34539"}
2024-08-09T11:51:43.670Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "ad155432-a4ce-459d-b532-272d494dd4f9"}
2024-08-09T11:51:49.105Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "f4060c54-c6b2-4d70-a2ae-6725393b6696"}
2024-08-09T11:51:54.382Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "cafb96a5-833f-4619-a72a-b3af1d672c99"}
2024-08-09T11:51:59.473Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "e223633b-4aa9-498a-b8e2-160b41e4deb1"}
2024-08-09T11:51:59.727Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "9e3653a4-2931-4535-a853-75e998630837"}
2024-08-09T11:52:04.679Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "9ed709f1-5823-4435-a713-ad71a297fa2d"}
2024-08-09T11:52:09.900Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "e957efde-8b65-459f-89a7-8aa86441444b"}
2024-08-09T11:52:15.092Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "7907cb2b-214c-46cd-a61e-55115f84057f"}
2024-08-09T11:52:20.279Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "2262da8d-a107-4a78-9561-72815cfd9a7c"}
2024-08-09T11:52:25.407Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "465ed199-91d7-4daf-9613-2af8d417933e"}
2024-08-09T11:54:19.179Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d70d961d-d81b-4eeb-aaba-7fb0e3ee900b", "new version": "8.0.19-10"}
2024-08-09T11:55:25.171Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4504ce0f-7f07-4d1d-97e0-30c1139bc1af", "object": "smart-update-pxc"}
2024-08-09T11:55:26.282Z	INFO	statefulSet was changed, run smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4504ce0f-7f07-4d1d-97e0-30c1139bc1af"}
2024-08-09T11:55:26.294Z	INFO	apply changes to secondary pod	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4504ce0f-7f07-4d1d-97e0-30c1139bc1af", "pod name": "smart-update-pxc-2"}
2024-08-09T11:55:26.294Z	INFO	primary pod	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4504ce0f-7f07-4d1d-97e0-30c1139bc1af", "pod name": "smart-update-pxc-0.smart-update-pxc.smart-update1-13135"}
2024-08-09T11:55:46.542Z	INFO	pod is waiting	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4504ce0f-7f07-4d1d-97e0-30c1139bc1af", "pod name": "smart-update-pxc-2", "reason": "PodInitializing"}
2024-08-09T11:56:46.587Z	INFO	pod is running	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4504ce0f-7f07-4d1d-97e0-30c1139bc1af", "pod name": "smart-update-pxc-2"}
2024-08-09T11:56:46.603Z	INFO	apply changes to secondary pod	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4504ce0f-7f07-4d1d-97e0-30c1139bc1af", "pod name": "smart-update-pxc-1"}
2024-08-09T11:58:06.942Z	INFO	pod is running	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4504ce0f-7f07-4d1d-97e0-30c1139bc1af", "pod name": "smart-update-pxc-1"}
2024-08-09T11:58:06.960Z	INFO	apply changes to primary pod	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4504ce0f-7f07-4d1d-97e0-30c1139bc1af", "pod name": "smart-update-pxc-0"}
2024-08-09T11:58:27.212Z	INFO	pod is waiting	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4504ce0f-7f07-4d1d-97e0-30c1139bc1af", "pod name": "smart-update-pxc-0", "reason": "PodInitializing"}
2024-08-09T11:59:27.241Z	INFO	pod is running	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4504ce0f-7f07-4d1d-97e0-30c1139bc1af", "pod name": "smart-update-pxc-0"}
2024-08-09T11:59:27.261Z	INFO	smart update finished	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4504ce0f-7f07-4d1d-97e0-30c1139bc1af"}
2024-08-09T11:59:27.345Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "4504ce0f-7f07-4d1d-97e0-30c1139bc1af", "new version": "8.0.36-28.1"}
2024-08-09T12:00:38.600Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "34be71b5-6096-4c1e-8526-18cf693353ea", "object": "smart-update-pxc"}
2024-08-09T12:00:38.697Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "34be71b5-6096-4c1e-8526-18cf693353ea", "object": "smart-update-haproxy"}
2024-08-09T12:00:38.742Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "34be71b5-6096-4c1e-8526-18cf693353ea", "object": "smart-update-pxc"}
2024-08-09T12:00:38.742Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "34be71b5-6096-4c1e-8526-18cf693353ea"}
2024-08-09T12:00:38.857Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "34be71b5-6096-4c1e-8526-18cf693353ea", "object": "smart-update-pxc-unready"}
2024-08-09T12:00:38.996Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "34be71b5-6096-4c1e-8526-18cf693353ea", "object": "smart-update-haproxy"}
2024-08-09T12:00:39.079Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "34be71b5-6096-4c1e-8526-18cf693353ea", "object": "smart-update-haproxy"}
2024-08-09T12:00:39.383Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "34be71b5-6096-4c1e-8526-18cf693353ea", "object": "smart-update-haproxy-replicas"}
2024-08-09T12:00:39.529Z	INFO	KubeAPIWarningLogger	.metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 232f9e28-771d-4a5f-96d3-186e3eebdcb0
2024-08-09T12:00:39.651Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "8b2641bc-176c-4b52-8cbb-dcd7546f9015", "object": "smart-update-pxc"}
2024-08-09T12:00:39.693Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "8b2641bc-176c-4b52-8cbb-dcd7546f9015"}
2024-08-09T12:00:40.884Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "0bf3824c-d702-4deb-8e5d-8f0ea98d2453"}
2024-08-09T12:00:45.994Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "a85eb46d-0af6-4f5c-90ba-7950efb55f88"}
2024-08-09T12:00:51.090Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "1041e22c-bb90-46b3-bfde-2bf6d9c14d51"}
2024-08-09T12:00:56.175Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "bef9bb9b-e175-4f43-9bae-ab4b70c0aa85"}
2024-08-09T12:01:01.249Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "23735f79-8c5f-4cae-ac97-fdec89a4e842"}
2024-08-09T12:01:06.335Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "3fa4073f-609a-4400-b100-3275eeff5fb5"}
2024-08-09T12:01:11.416Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "b69abc41-902f-42f3-a137-2fc074a69f38"}
2024-08-09T12:01:16.483Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "e9d1d186-e735-4892-b969-09bdb4efb45e"}
2024-08-09T12:01:21.547Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "46f298f5-f71b-40e5-911d-e3834d46fe4a"}
2024-08-09T12:01:26.610Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "3e9f8d1a-1a4a-4349-a690-fe1b62f67322"}
2024-08-09T12:01:31.690Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "391d48f8-29af-4b27-b42b-7a34d703a38a"}
2024-08-09T12:01:36.755Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "537b6ec7-5795-4584-b6a5-bd9edc819038"}
2024-08-09T12:01:41.817Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "88ba56be-e30c-4ef8-99e1-6c87c84b1897"}
2024-08-09T12:01:46.876Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "c60a466a-75ec-4f1a-ad30-124e29783ca6"}
2024-08-09T12:01:51.953Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "cd53de4a-c50a-408f-868f-71a7676d90bd"}
2024-08-09T12:01:57.037Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "f1c4b9c5-f868-4676-ba31-5f7cfbad255d"}
2024-08-09T12:01:57.191Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "2bb982f7-31e9-443e-9806-cab46ee919fc"}
2024-08-09T12:02:02.194Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "5dfc5e2d-31d7-44bf-b858-b9c406c1f6e2"}
2024-08-09T12:02:07.404Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "3b89394e-dbac-45bf-a269-d3ac5e45e3d1"}
2024-08-09T12:02:12.537Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "89e07db0-fafb-4ba6-9303-617119a79cc3"}
2024-08-09T12:02:17.705Z	DEBUG	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "0d547287-5963-4215-bacd-10b88bdb1023"}
2024-08-09T12:04:23.405Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "e507bd0b-6512-478f-a264-99662e052730", "new version": "8.0.19-10"}
2024-08-09T12:05:29.721Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d086c428-85e6-410d-9171-656d73503823", "object": "smart-update-pxc"}
2024-08-09T12:05:30.781Z	INFO	statefulSet was changed, run smart update	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d086c428-85e6-410d-9171-656d73503823"}
2024-08-09T12:05:30.792Z	INFO	apply changes to secondary pod	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d086c428-85e6-410d-9171-656d73503823", "pod name": "smart-update-pxc-2"}
2024-08-09T12:05:30.792Z	INFO	primary pod	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d086c428-85e6-410d-9171-656d73503823", "pod name": "smart-update-pxc-0.smart-update-pxc.smart-update1-13135"}
2024-08-09T12:06:51.106Z	INFO	pod is running	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d086c428-85e6-410d-9171-656d73503823", "pod name": "smart-update-pxc-2"}
2024-08-09T12:06:51.123Z	INFO	apply changes to secondary pod	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d086c428-85e6-410d-9171-656d73503823", "pod name": "smart-update-pxc-1"}
2024-08-09T12:07:11.351Z	INFO	pod is waiting	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d086c428-85e6-410d-9171-656d73503823", "pod name": "smart-update-pxc-1", "reason": "PodInitializing"}
2024-08-09T12:08:21.384Z	INFO	pod is running	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d086c428-85e6-410d-9171-656d73503823", "pod name": "smart-update-pxc-1"}
2024-08-09T12:08:21.401Z	INFO	apply changes to primary pod	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d086c428-85e6-410d-9171-656d73503823", "pod name": "smart-update-pxc-0"}
2024-08-09T12:08:41.666Z	INFO	pod is waiting	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d086c428-85e6-410d-9171-656d73503823", "pod name": "smart-update-pxc-0", "reason": "PodInitializing"}
2024-08-09T12:09:41.679Z	INFO	pod is running	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d086c428-85e6-410d-9171-656d73503823", "pod name": "smart-update-pxc-0"}
2024-08-09T12:09:41.706Z	INFO	smart update finished	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d086c428-85e6-410d-9171-656d73503823"}
2024-08-09T12:09:41.810Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "smart-update1-13135", "name": "smart-update", "reconcileID": "d086c428-85e6-410d-9171-656d73503823", "new version": "8.0.36-28.1"}
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.NctDChkrxi
++ mktemp
+ local LAST_ERR=/tmp/tmp.XpZN3nvDnD
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.NctDChkrxi
No resources found
+ cat /tmp/tmp.XpZN3nvDnD
+ rm /tmp/tmp.NctDChkrxi /tmp/tmp.XpZN3nvDnD
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.MkMDvgxfm5
++ mktemp
+ local LAST_ERR=/tmp/tmp.e0twahmUsY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MkMDvgxfm5
No resources found
+ cat /tmp/tmp.e0twahmUsY
+ rm /tmp/tmp.MkMDvgxfm5 /tmp/tmp.e0twahmUsY
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.0DaQTVAp2S
++ mktemp
+ local LAST_ERR=/tmp/tmp.0fuzBAMucn
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.0DaQTVAp2S
No resources found
+ cat /tmp/tmp.0fuzBAMucn
+ rm /tmp/tmp.0DaQTVAp2S /tmp/tmp.0fuzBAMucn
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.t4ZHTrYKaX
++ mktemp
+ local LAST_ERR=/tmp/tmp.HeT2rcBzjr
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.t4ZHTrYKaX
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.HeT2rcBzjr
+ rm /tmp/tmp.t4ZHTrYKaX /tmp/tmp.HeT2rcBzjr
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace smart-update1-13135
+ rm -rf /tmp/tmp.c7lkEUIUkH
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.REhRgLjXFm
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

++ mktemp
+ local LAST_OUT=/tmp/tmp.FhRh3hkdQO
+ local LAST_ERR=/tmp/tmp.wGLGZHTpym
+ local exit_status=0
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.JnsFYgvWaG
+ local exit_status=0
+ for i in '$(seq 0 2)'
+ set +e
++ seq 0 2
+ kubectl delete --grace-period=0 --force=true namespace smart-update1-13135
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator