Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/logs/smart-update2-8-4.log
Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1
+ API=pxc.percona.com/v9-9-9
+ TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.4
+ CLUSTER=smart-update
+ CLUSTER_SIZE=3
+ PROXY_SIZE=2
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]]
++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/'
++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.4
+ PXC_VER=8.4
+ TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.4
+ VS_URL=http://version-service
+ VS_PORT=11000
+ VS_ENDPOINT=http://version-service:11000
+ [[ 8.4 == \8\.\4 ]]
+ VS_UPDATE_STRATEGY=latest
+ main
+ create_infra smart-update2-28785
+ local ns=smart-update2-28785
+ '[' -n pxc-operator ']'
+ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}'
+ read -r pns name
+ [[ -z smart-update2-8225 ]]
+ [[ -z minimal-cluster ]]
+ kubectl_bin patch pxc -n smart-update2-8225 minimal-cluster --type=merge -p '{"metadata":{"finalizers":[]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.rOOvGgaspV
++ mktemp
+ local LAST_ERR=/tmp/tmp.0QOlT9f8NV
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch pxc -n smart-update2-8225 minimal-cluster --type=merge -p '{"metadata":{"finalizers":[]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.rOOvGgaspV
perconaxtradbcluster.pxc.percona.com/minimal-cluster patched (no change)
+ cat /tmp/tmp.0QOlT9f8NV
+ rm /tmp/tmp.rOOvGgaspV /tmp/tmp.0QOlT9f8NV
+ return 0
+ read -r pns name
+ read -r pns name
+ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}'
+ timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found
perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-8225 namespace
+ timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found
No resources found
+ timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found
No resources found
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ grep chaos-mesh.org
++ awk '{print $1}'
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrole
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ xargs kubectl delete ns
+ awk '{print$1}'
++ mktemp
+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ local LAST_OUT=/tmp/tmp.e5BEKzB3wg
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.zDjZxvPPW4
+ local LAST_ERR=/tmp/tmp.QryOYWkEoL
+ local exit_status=0
++ mktemp
+ local LAST_ERR=/tmp/tmp.r1k7jL1CcO
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.zDjZxvPPW4
+ cat /tmp/tmp.r1k7jL1CcO
+ rm /tmp/tmp.zDjZxvPPW4 /tmp/tmp.r1k7jL1CcO
+ return 0
namespace "cert-manager" deleted
namespace "smart-update2-8225" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.e5BEKzB3wg
namespace "pxc-operator" deleted
+ cat /tmp/tmp.QryOYWkEoL
+ rm /tmp/tmp.e5BEKzB3wg /tmp/tmp.QryOYWkEoL
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.tgISl0j7UZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.nFSgntlRGt
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.tgISl0j7UZ
namespace/pxc-operator created
+ cat /tmp/tmp.nFSgntlRGt
+ rm /tmp/tmp.tgISl0j7UZ /tmp/tmp.nFSgntlRGt
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tC6mTOD2yf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0gH1rr8Ilf
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.tC6mTOD2yf
++ cat /tmp/tmp.0gH1rr8Ilf
++ rm /tmp/tmp.tC6mTOD2yf /tmp/tmp.0gH1rr8Ilf
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster3 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.gznxQZUqCc
++ mktemp
+ local LAST_ERR=/tmp/tmp.Lrxll83Q0q
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster3 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gznxQZUqCc
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster3" modified.
+ cat /tmp/tmp.Lrxll83Q0q
+ rm /tmp/tmp.gznxQZUqCc /tmp/tmp.Lrxll83Q0q
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.A4iIO4LJj1
++ mktemp
+ local LAST_ERR=/tmp/tmp.7SgjlHIpWP
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.A4iIO4LJj1
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.7SgjlHIpWP
+ rm /tmp/tmp.A4iIO4LJj1 /tmp/tmp.7SgjlHIpWP
+ return 0
+ kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.QCuCBoosC8
++ mktemp
+ local LAST_ERR=/tmp/tmp.e4GnxTKBeI
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QCuCBoosC8
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met
+ cat /tmp/tmp.e4GnxTKBeI
+ rm /tmp/tmp.QCuCBoosC8 /tmp/tmp.e4GnxTKBeI
+ return 0
+ kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.0bl9fYuIfT
++ mktemp
+ local LAST_ERR=/tmp/tmp.g0fECTHp3v
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.0bl9fYuIfT
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met
+ cat /tmp/tmp.g0fECTHp3v
+ rm /tmp/tmp.0bl9fYuIfT /tmp/tmp.g0fECTHp3v
+ return 0
+ kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.9ae1XbntZi
++ mktemp
+ local LAST_ERR=/tmp/tmp.gLQgh2VykO
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.9ae1XbntZi
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met
+ cat /tmp/tmp.gLQgh2VykO
+ rm /tmp/tmp.9ae1XbntZi /tmp/tmp.gLQgh2VykO
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-rbac.yaml
+ kubectl_bin apply -f -
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
++ mktemp
+ local LAST_OUT=/tmp/tmp.2XNCSQkRPX
++ mktemp
+ local LAST_ERR=/tmp/tmp.314VZwCi4s
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2XNCSQkRPX
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.314VZwCi4s
+ rm /tmp/tmp.2XNCSQkRPX /tmp/tmp.314VZwCi4s
+ return 0
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-operator.yaml
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' -
+ kubectl_bin apply -f -
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.tmTpgxHna5
++ mktemp
+ local LAST_ERR=/tmp/tmp.RgW5Dll1Iu
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.tmTpgxHna5
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.RgW5Dll1Iu
+ rm /tmp/tmp.tmTpgxHna5 /tmp/tmp.RgW5Dll1Iu
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.agJvafkL20
++ mktemp
+ local LAST_ERR=/tmp/tmp.Wo0eU71gaH
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.agJvafkL20
pod/percona-xtradb-cluster-operator-658fcf4b98-gkh9h condition met
+ cat /tmp/tmp.Wo0eU71gaH
+ rm /tmp/tmp.agJvafkL20 /tmp/tmp.Wo0eU71gaH
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ head -1
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pRqMNJPM0N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1DqqS8r21u
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pRqMNJPM0N
++ cat /tmp/tmp.1DqqS8r21u
++ rm /tmp/tmp.pRqMNJPM0N /tmp/tmp.1DqqS8r21u
++ return 0
+ wait_pod percona-xtradb-cluster-operator-658fcf4b98-gkh9h 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-658fcf4b98-gkh9h
+ local max_retry=480
+ local ns=pxc-operator
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo percona-xtradb-cluster-operator-658fcf4b98-gkh9h
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-658fcf4b98-gkh9h condition met
waiting for pod/percona-xtradb-cluster-operator-658fcf4b98-gkh9h to become Ready.Ok
+ sleep 3
+ create_namespace smart-update2-28785
+ local namespace=smart-update2-28785
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl api-resources
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces smart-update2-28785'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces smart-update2-28785
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace smart-update2-28785
+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ xargs kubectl delete ns
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.4cLvg36q2S
+ local LAST_OUT=/tmp/tmp.I1ZM0E2WWb
++ mktemp
+ local LAST_ERR=/tmp/tmp.UY10PEbl1s
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.oUOl3by1TA
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace smart-update2-28785
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace smart-update2-28785
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.I1ZM0E2WWb
+ cat /tmp/tmp.UY10PEbl1s
+ rm /tmp/tmp.I1ZM0E2WWb /tmp/tmp.UY10PEbl1s
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace smart-update2-28785
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.4cLvg36q2S
+ cat /tmp/tmp.oUOl3by1TA
Error from server (NotFound): namespaces "smart-update2-28785" not found
+ rm /tmp/tmp.4cLvg36q2S /tmp/tmp.oUOl3by1TA
+ return 1
+ :
+ wait_for_delete namespace/smart-update2-28785
+ local res=namespace/smart-update2-28785
+ echo -n 'waiting for namespace/smart-update2-28785 to be deleted'
waiting for namespace/smart-update2-28785 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "smart-update2-28785" not found
+ desc 'create namespace smart-update2-28785'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace smart-update2-28785
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace smart-update2-28785
++ mktemp
+ local LAST_OUT=/tmp/tmp.7UMlJDhoOY
++ mktemp
+ local LAST_ERR=/tmp/tmp.WIsuwny4Cb
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace smart-update2-28785
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7UMlJDhoOY
namespace/smart-update2-28785 created
+ cat /tmp/tmp.WIsuwny4Cb
+ rm /tmp/tmp.7UMlJDhoOY /tmp/tmp.WIsuwny4Cb
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bSWMca3cti
+++ mktemp
++ local LAST_ERR=/tmp/tmp.x3YVznwldw
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bSWMca3cti
++ cat /tmp/tmp.x3YVznwldw
++ rm /tmp/tmp.bSWMca3cti /tmp/tmp.x3YVznwldw
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster3 --namespace=smart-update2-28785
++ mktemp
+ local LAST_OUT=/tmp/tmp.nHdaLqagCb
++ mktemp
+ local LAST_ERR=/tmp/tmp.EOYYMRWwmr
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster3 --namespace=smart-update2-28785
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nHdaLqagCb
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster3" modified.
+ cat /tmp/tmp.EOYYMRWwmr
+ rm /tmp/tmp.nHdaLqagCb /tmp/tmp.EOYYMRWwmr
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.RwZrktS3Tq
++ mktemp
+ local LAST_ERR=/tmp/tmp.O97GVroZre
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.RwZrktS3Tq
secret/minio-secret created
secret/aws-s3-secret created
secret/do-spaces-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.O97GVroZre
+ rm /tmp/tmp.RwZrktS3Tq /tmp/tmp.O97GVroZre
+ return 0
+ deploy_version_service
+ desc 'install version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install version service
-----------------------------------------------------------------------------------

+ kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json
++ mktemp
+ local LAST_OUT=/tmp/tmp.MHviu0RhnC
++ mktemp
+ local LAST_ERR=/tmp/tmp.1rNoLqJzQS
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MHviu0RhnC
configmap/versions created
+ cat /tmp/tmp.1rNoLqJzQS
+ rm /tmp/tmp.MHviu0RhnC /tmp/tmp.1rNoLqJzQS
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/vs.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.OumI1avwgH
++ mktemp
+ local LAST_ERR=/tmp/tmp.U4q2ic2Tck
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/vs.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.OumI1avwgH
deployment.apps/version-service created
service/version-service created
+ cat /tmp/tmp.U4q2ic2Tck
+ rm /tmp/tmp.OumI1avwgH /tmp/tmp.U4q2ic2Tck
+ return 0
+ sleep 10
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.Hd78SaU2mO
++ mktemp
+ local LAST_ERR=/tmp/tmp.WH8fyZzGY7
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Hd78SaU2mO
namespace/cert-manager created
+ cat /tmp/tmp.WH8fyZzGY7
+ rm /tmp/tmp.Hd78SaU2mO /tmp/tmp.WH8fyZzGY7
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.qO8lGvouW0
++ mktemp
+ local LAST_ERR=/tmp/tmp.kFfwbgbMku
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.qO8lGvouW0
namespace/cert-manager labeled
+ cat /tmp/tmp.kFfwbgbMku
+ rm /tmp/tmp.qO8lGvouW0 /tmp/tmp.kFfwbgbMku
+ return 0
+ kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.JE4XAvPMaN
++ mktemp
+ local LAST_ERR=/tmp/tmp.eJctCACR4m
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.JE4XAvPMaN
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-tokenrequest created
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager-cainjector created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.eJctCACR4m
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.JE4XAvPMaN /tmp/tmp.eJctCACR4m
+ return 0
+ '[' '' == 4.10 ']'
+ kubectl_bin -n cert-manager rollout status deployment/cert-manager --timeout=120s
++ mktemp
+ local LAST_OUT=/tmp/tmp.lOt8SIwVon
++ mktemp
+ local LAST_ERR=/tmp/tmp.IxX4gRieGX
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl -n cert-manager rollout status deployment/cert-manager --timeout=120s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lOt8SIwVon
deployment "cert-manager" successfully rolled out
+ cat /tmp/tmp.IxX4gRieGX
+ rm /tmp/tmp.lOt8SIwVon /tmp/tmp.IxX4gRieGX
+ return 0
+ kubectl_bin -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s
++ mktemp
+ local LAST_OUT=/tmp/tmp.K2DSqGx1Mw
++ mktemp
+ local LAST_ERR=/tmp/tmp.pg2uHArSK8
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.K2DSqGx1Mw
deployment "cert-manager-cainjector" successfully rolled out
+ cat /tmp/tmp.pg2uHArSK8
+ rm /tmp/tmp.K2DSqGx1Mw /tmp/tmp.pg2uHArSK8
+ return 0
+ kubectl_bin -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s
++ mktemp
+ local LAST_OUT=/tmp/tmp.WRnAxJs744
++ mktemp
+ local LAST_ERR=/tmp/tmp.vDu8JgJryp
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.WRnAxJs744
Waiting for deployment "cert-manager-webhook" rollout to finish: 0 of 1 updated replicas are available...
deployment "cert-manager-webhook" successfully rolled out
+ cat /tmp/tmp.vDu8JgJryp
+ rm /tmp/tmp.WRnAxJs744 /tmp/tmp.vDu8JgJryp
+ return 0
+ log 'waiting for cert-manager webhook caBundle to be injected'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-10-05T14:31:10+0000]' waiting for cert-manager webhook caBundle to be injected
[2026-10-05T14:31:10+0000] waiting for cert-manager webhook caBundle to be injected
++ date +%s
+ local deadline=1791210790
+ grep -q .
+ kubectl_bin get validatingwebhookconfigurations cert-manager-webhook -o 'jsonpath={.webhooks[0].clientConfig.caBundle}'
+ log 'probing cert-manager webhook'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-10-05T14:31:12+0000]' probing cert-manager webhook
[2026-10-05T14:31:12+0000] probing cert-manager webhook
+ local probe_file
++ mktemp
+ probe_file=/tmp/tmp.lnksfJr4wA
+ cat
+ local retries=0
+ kubectl_bin apply -f /tmp/tmp.lnksfJr4wA --dry-run=server
+ rm -f /tmp/tmp.lnksfJr4wA
++ sort -V
++ tail -n3
++ head -n1
++ grep :8.4
++ jq -r '.versions[].matrix.pxc[].imagePath'
+++ get_operator_pod
+++ local label_prefix=app.kubernetes.io/
++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++++ grep -c percona-xtradb-cluster-operator
+++ local check_label=1
+++ [[ 1 -eq 0 ]]
+++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
+++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ head -1
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.i3nhYMEJyd
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KnOgW27tdR
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.i3nhYMEJyd
+++ cat /tmp/tmp.KnOgW27tdR
+++ rm /tmp/tmp.i3nhYMEJyd /tmp/tmp.KnOgW27tdR
+++ return 0
++ kubectl_bin exec -ti percona-xtradb-cluster-operator-658fcf4b98-gkh9h -n pxc-operator -- curl -s http://version-service.smart-update2-28785.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mKSnibJDqf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2ibnapFiMY
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec -ti percona-xtradb-cluster-operator-658fcf4b98-gkh9h -n pxc-operator -- curl -s http://version-service.smart-update2-28785.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.mKSnibJDqf
++ cat /tmp/tmp.2ibnapFiMY
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.mKSnibJDqf /tmp/tmp.2ibnapFiMY
++ return 0
+ IMAGE_PXC=percona/percona-xtradb-cluster:8.4.5-5.1
+ kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.cTk46nCfA1
++ mktemp
+ local LAST_ERR=/tmp/tmp.BPMUnZPbNC
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.cTk46nCfA1
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched
+ cat /tmp/tmp.BPMUnZPbNC
+ rm /tmp/tmp.cTk46nCfA1 /tmp/tmp.BPMUnZPbNC
+ return 0
+ kubectl_bin -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-28785.svc.cluster.local:11000
++ mktemp
+ local LAST_OUT=/tmp/tmp.vWEcDIvAWL
++ mktemp
+ local LAST_ERR=/tmp/tmp.sK6gNtX7fj
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-28785.svc.cluster.local:11000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vWEcDIvAWL
deployment.apps/percona-xtradb-cluster-operator env updated
+ cat /tmp/tmp.sK6gNtX7fj
+ rm /tmp/tmp.vWEcDIvAWL /tmp/tmp.sK6gNtX7fj
+ return 0
+ desc 'Starting telemetry testing'
+ set +o xtrace


-----------------------------------------------------------------------------------
Starting telemetry testing
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f -
+ /usr/bin/sed s/version-service/version-service-cr/g /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/vs.yml
++ mktemp
++ yq 'select(.kind == "Deployment").spec.template.spec.containers[0].image' /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/vs.yml
+ local LAST_OUT=/tmp/tmp.XMS8QMljpr
++ mktemp
+ local LAST_ERR=/tmp/tmp.RFk2W4wa5U
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ yq eval '(. | select(.kind == "Deployment") | .spec.template.spec.containers[0].image) = "perconalab/version-service:main-latest"'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XMS8QMljpr
deployment.apps/version-service-cr created
service/version-service-cr created
+ cat /tmp/tmp.RFk2W4wa5U
+ rm /tmp/tmp.XMS8QMljpr /tmp/tmp.RFk2W4wa5U
+ return 0
+ kubectl_bin delete pod -l run=version-service
++ mktemp
+ local LAST_OUT=/tmp/tmp.bjXMqFRoyJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.TE3IfReTqY
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pod -l run=version-service
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.bjXMqFRoyJ
pod "version-service-66774c865b-jsmqz" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.TE3IfReTqY
+ rm /tmp/tmp.bjXMqFRoyJ /tmp/tmp.TE3IfReTqY
+ return 0
++ sed -r 's/^.*:([0-9]+.[0-9]+).*/\1/'
++ echo -n percona/percona-xtradb-cluster:8.4.5-5.1
+ IMAGE_PREFIX=8.4
+ desc 'Enable telemetry on operator level'
+ set +o xtrace


-----------------------------------------------------------------------------------
Enable telemetry on operator level
-----------------------------------------------------------------------------------

+ yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "false"'
+ kubectl_bin apply -n pxc-operator -f -
+ kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.lN6s3kFsqo
++ mktemp
+ local LAST_OUT=/tmp/tmp.dsVbrNm847
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.3JsFnhsKmD
+ local exit_status=0
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.3uuIEFeWQP
+ local exit_status=0
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n pxc-operator -f -
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.dsVbrNm847
+ cat /tmp/tmp.3uuIEFeWQP
+ rm /tmp/tmp.dsVbrNm847 /tmp/tmp.3uuIEFeWQP
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lN6s3kFsqo
deployment.apps/percona-xtradb-cluster-operator configured
+ cat /tmp/tmp.3JsFnhsKmD
+ rm /tmp/tmp.lN6s3kFsqo /tmp/tmp.3JsFnhsKmD
+ return 0
+ sleep 30
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ head -1
+++ mktemp
++ local LAST_OUT=/tmp/tmp.O66ADDSGWt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dJzEWPkFGJ
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.O66ADDSGWt
++ cat /tmp/tmp.dJzEWPkFGJ
++ rm /tmp/tmp.O66ADDSGWt /tmp/tmp.dJzEWPkFGJ
++ return 0
+ wait_pod percona-xtradb-cluster-operator-b7465f595-bbxn6 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-b7465f595-bbxn6
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-b7465f595-bbxn6
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-b7465f595-bbxn6 condition met
waiting for pod/percona-xtradb-cluster-operator-b7465f595-bbxn6 to become Ready.Ok
+ check_telemetry_transfer http://version-service-cr.smart-update2-28785.svc.cluster.local:11000 disabled enabled
+ local cr_vs_uri=http://version-service-cr.smart-update2-28785.svc.cluster.local:11000
+ local cr_vs_channel=disabled
+ local telemetry_state=enabled
+ desc 'create PXC minimal cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC minimal cluster
-----------------------------------------------------------------------------------

+ cluster=minimal-cluster
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.xP2KiJWGxP
++ mktemp
+ local LAST_ERR=/tmp/tmp.aUMWlJK0Cg
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xP2KiJWGxP
deployment.apps/pxc-client created
+ cat /tmp/tmp.aUMWlJK0Cg
+ rm /tmp/tmp.xP2KiJWGxP /tmp/tmp.aUMWlJK0Cg
+ return 0
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml
+ kubectl_bin apply -f -
+ yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.z4FRQmvesG
++ mktemp
+ local LAST_ERR=/tmp/tmp.f1Irjx8lsm
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.z4FRQmvesG
secret/minimal-cluster created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.f1Irjx8lsm
+ rm /tmp/tmp.z4FRQmvesG /tmp/tmp.f1Irjx8lsm
+ return 0
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cr-minimal.yaml
+ yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-28785.svc.cluster.local:11000"'
+ yq eval '.spec.crVersion="9.9.9"'
+ yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.4.5-5.1"'
+ yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"'
+ yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"'
+ yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420"'
+ yq eval '.spec.upgradeOptions.apply="disabled"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.lRRkSiAjjR
++ mktemp
+ local LAST_ERR=/tmp/tmp.MPXAutlN91
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lRRkSiAjjR
perconaxtradbcluster.pxc.percona.com/minimal-cluster created
+ cat /tmp/tmp.MPXAutlN91
+ rm /tmp/tmp.lRRkSiAjjR /tmp/tmp.MPXAutlN91
+ return 0
+ desc 'check if Pod is started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod is started
-----------------------------------------------------------------------------------

+ wait_for_running minimal-cluster-pxc 1
+ local name=minimal-cluster-pxc
+ local size=1
+ local last_pod
+ local max_retry=480
+ [[ -z minimal-cluster-pxc ]]
+ [[ 1 -le 0 ]]
+ last_pod=0
+ desc 'Waiting for 1 running pods for minimal-cluster-pxc'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 1 running pods for minimal-cluster-pxc
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in $(seq 0 $last_pod)
+ wait_pod minimal-cluster-pxc-0 480
+ local pod=minimal-cluster-pxc-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/minimal-cluster-pxc-0 condition met
waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok
+ sleep 20
+ local proxy
++ get_proxy minimal-cluster
++ local target_cluster=minimal-cluster
+++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.H6orQRycn5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fvsCubfswO
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.H6orQRycn5
+++ cat /tmp/tmp.fvsCubfswO
+++ rm /tmp/tmp.H6orQRycn5 /tmp/tmp.fvsCubfswO
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo minimal-cluster-haproxy
++ return
+ proxy=minimal-cluster-haproxy
+ wait_for_running minimal-cluster-haproxy 1
+ local name=minimal-cluster-haproxy
+ local size=1
+ local last_pod
+ local max_retry=480
+ [[ -z minimal-cluster-haproxy ]]
+ [[ 1 -le 0 ]]
+ last_pod=0
+ desc 'Waiting for 1 running pods for minimal-cluster-haproxy'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 1 running pods for minimal-cluster-haproxy
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in $(seq 0 $last_pod)
+ wait_pod minimal-cluster-haproxy-0 480
+ local pod=minimal-cluster-haproxy-0
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo minimal-cluster-haproxy-0
+ local container=
+ set +o xtrace
pod/minimal-cluster-haproxy-0 condition met
waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.o47ZJH07Qr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aOPrKTH0PX
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.o47ZJH07Qr
++ cat /tmp/tmp.aOPrKTH0PX
++ rm /tmp/tmp.o47ZJH07Qr /tmp/tmp.aOPrKTH0PX
++ return 0
+ client_pod=pxc-client-6fc75f888d-6xnhn
+ wait_pod pxc-client-6fc75f888d-6xnhn
+ local pod=pxc-client-6fc75f888d-6xnhn
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ echo pxc-client-6fc75f888d-6xnhn
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6fc75f888d-6xnhn condition met
waiting for pod/pxc-client-6fc75f888d-6xnhn to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument.
command terminated with exit code 5
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Vg34ZwfVi4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r3SJmhEOCo
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Vg34ZwfVi4
++ cat /tmp/tmp.r3SJmhEOCo
++ rm /tmp/tmp.Vg34ZwfVi4 /tmp/tmp.r3SJmhEOCo
++ return 0
+ client_pod=pxc-client-6fc75f888d-6xnhn
+ wait_pod pxc-client-6fc75f888d-6xnhn
+ local pod=pxc-client-6fc75f888d-6xnhn
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo pxc-client-6fc75f888d-6xnhn
+ local container=
+ set +o xtrace
pod/pxc-client-6fc75f888d-6xnhn condition met
waiting for pod/pxc-client-6fc75f888d-6xnhn to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument.
command terminated with exit code 5
+ jq 'del(."grpc.time_ms")'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ grep -Eo '\{.*\}'
+ grep -E 'server request payload|unary call'
++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}'
+ kubectl_bin logs version-service-cr-58b8ffbbb4-jbf97
++ mktemp
+ local LAST_OUT=/tmp/tmp.wPTmE1Ke7P
++ mktemp
+ local LAST_ERR=/tmp/tmp.xPTpOH3B4E
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs version-service-cr-58b8ffbbb4-jbf97
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.wPTmE1Ke7P
+ cat /tmp/tmp.xPTpOH3B4E
+ rm /tmp/tmp.wPTmE1Ke7P /tmp/tmp.xPTpOH3B4E
+ return 0
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.time_ms")'
+ grep -E 'server request payload|unary call'
+ grep -Eo '\{.*\}'
++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}'
+ kubectl_bin logs version-service-66774c865b-j48hf
++ mktemp
+ local LAST_OUT=/tmp/tmp.irNEmLaqk2
++ mktemp
+ local LAST_ERR=/tmp/tmp.Us1di9Pjv8
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs version-service-66774c865b-j48hf
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.irNEmLaqk2
+ cat /tmp/tmp.Us1di9Pjv8
+ rm /tmp/tmp.irNEmLaqk2 /tmp/tmp.Us1di9Pjv8
+ return 0
+ local telemetry_log_file=enabled_telemetry.version-service-cw.log.json
+ desc 'telemetry was disabled in CR but in operator not'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR but in operator not
-----------------------------------------------------------------------------------

+ '[' disabled == disabled -a enabled == enabled ']'
+ desc 'operator fallback VS should have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
operator fallback VS should have telemetry
-----------------------------------------------------------------------------------

+ diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json /dev/fd/63
++ grep -f /tmp/tmp.kxnkOjofaU/enabled_telemetry.version-service.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json
+ desc 'CR VS should not have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
CR VS should not have telemetry
-----------------------------------------------------------------------------------

+ [[ -s /tmp/tmp.kxnkOjofaU/enabled_telemetry.version-service-cr.log.json ]]
+ local image_prefix=disabled
+ image_prefix=disabled
+ local telemetry_cr_log_file=enabled_telemetry.version-service-cr-disabled-cw.log.json
+ desc 'telemetry was disabled in operator but not in CR'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in operator but not in CR
-----------------------------------------------------------------------------------

+ [[ disabled == \d\i\s\a\b\l\e\d\-\r\e\c\o\m\m\e\n\d\e\d ]]
+ [[ disabled == \d\i\s\a\b\l\e\d\-\l\a\t\e\s\t ]]
+ desc 'telemetry was disabled in CR as well as in operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR as well as in operator
-----------------------------------------------------------------------------------

+ '[' disabled == disabled -a enabled == disabled ']'
+ kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.3m9Z7cyYIa
++ mktemp
+ local LAST_ERR=/tmp/tmp.glpQ7gg7wQ
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.3m9Z7cyYIa
perconaxtradbcluster.pxc.percona.com/minimal-cluster patched
+ cat /tmp/tmp.glpQ7gg7wQ
+ rm /tmp/tmp.3m9Z7cyYIa /tmp/tmp.glpQ7gg7wQ
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ head -1
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4jntD9BvWO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cQk7xLu39N
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4jntD9BvWO
++ cat /tmp/tmp.cQk7xLu39N
++ rm /tmp/tmp.4jntD9BvWO /tmp/tmp.cQk7xLu39N
++ return 0
+ kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-b7465f595-bbxn6
++ mktemp
+ local LAST_OUT=/tmp/tmp.TnFA5pJ0ZD
++ mktemp
+ local LAST_ERR=/tmp/tmp.LYv3bAIVtN
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-b7465f595-bbxn6
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.TnFA5pJ0ZD
pod "percona-xtradb-cluster-operator-b7465f595-bbxn6" deleted from pxc-operator namespace
+ cat /tmp/tmp.LYv3bAIVtN
+ rm /tmp/tmp.TnFA5pJ0ZD /tmp/tmp.LYv3bAIVtN
+ return 0
+ kubectl_bin delete pxc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.QWjq3IN0Dd
++ mktemp
+ local LAST_ERR=/tmp/tmp.DIySAn6tRx
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pxc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QWjq3IN0Dd
perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.DIySAn6tRx
+ rm /tmp/tmp.QWjq3IN0Dd /tmp/tmp.DIySAn6tRx
+ return 0
+ kubectl_bin delete deploy pxc-client
++ mktemp
+ local LAST_OUT=/tmp/tmp.wnEss2S03S
++ mktemp
+ local LAST_ERR=/tmp/tmp.Zk2TPDIW3J
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete deploy pxc-client
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.wnEss2S03S
deployment.apps "pxc-client" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.Zk2TPDIW3J
+ rm /tmp/tmp.wnEss2S03S /tmp/tmp.Zk2TPDIW3J
+ return 0
+ sleep 30
+ desc 'Disabling telemetry on the operator level'
+ set +o xtrace


-----------------------------------------------------------------------------------
Disabling telemetry on the operator level
-----------------------------------------------------------------------------------

+ kubectl_bin delete pod -l run=version-service-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.m6W7IAGZrT
++ mktemp
+ local LAST_ERR=/tmp/tmp.g28sTfTvx3
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pod -l run=version-service-cr
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.m6W7IAGZrT
pod "version-service-cr-58b8ffbbb4-jbf97" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.g28sTfTvx3
+ rm /tmp/tmp.m6W7IAGZrT /tmp/tmp.g28sTfTvx3
+ return 0
+ kubectl_bin delete pod -l run=version-service
++ mktemp
+ local LAST_OUT=/tmp/tmp.yMXYdQAsNw
++ mktemp
+ local LAST_ERR=/tmp/tmp.mEFbpyLjzG
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pod -l run=version-service
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.yMXYdQAsNw
pod "version-service-66774c865b-j48hf" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.mEFbpyLjzG
+ rm /tmp/tmp.yMXYdQAsNw /tmp/tmp.mEFbpyLjzG
+ return 0
+ kubectl_bin apply -n pxc-operator -f -
+ yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.IPfzSl7P5N
+ kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator
++ mktemp
+ local LAST_ERR=/tmp/tmp.sRcYmQ7r9g
+ local exit_status=0
++ seq 0 2
++ mktemp
+ local LAST_OUT=/tmp/tmp.48Uws5RrH7
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n pxc-operator -f -
++ mktemp
+ local LAST_ERR=/tmp/tmp.14jFxTZ7Y1
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.48Uws5RrH7
+ cat /tmp/tmp.14jFxTZ7Y1
+ rm /tmp/tmp.48Uws5RrH7 /tmp/tmp.14jFxTZ7Y1
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.IPfzSl7P5N
deployment.apps/percona-xtradb-cluster-operator configured
+ cat /tmp/tmp.sRcYmQ7r9g
+ rm /tmp/tmp.IPfzSl7P5N /tmp/tmp.sRcYmQ7r9g
+ return 0
+ sleep 30
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ head -1
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RpWzB1gSxP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wWXphnBUOJ
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RpWzB1gSxP
++ cat /tmp/tmp.wWXphnBUOJ
++ rm /tmp/tmp.RpWzB1gSxP /tmp/tmp.wWXphnBUOJ
++ return 0
+ wait_pod percona-xtradb-cluster-operator-559d7d8944-b58tt 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-559d7d8944-b58tt
+ local max_retry=480
+ local ns=pxc-operator
++ grep -E '^(pxc|proxysql)$'
++ echo percona-xtradb-cluster-operator-559d7d8944-b58tt
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-559d7d8944-b58tt condition met
waiting for pod/percona-xtradb-cluster-operator-559d7d8944-b58tt to become Ready.Ok
+ check_telemetry_transfer http://version-service-cr.smart-update2-28785.svc.cluster.local:11000 8.4-latest disabled
+ local cr_vs_uri=http://version-service-cr.smart-update2-28785.svc.cluster.local:11000
+ local cr_vs_channel=8.4-latest
+ local telemetry_state=disabled
+ desc 'create PXC minimal cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC minimal cluster
-----------------------------------------------------------------------------------

+ cluster=minimal-cluster
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.C2XHSIwH6D
++ mktemp
+ local LAST_ERR=/tmp/tmp.csyFaII3GM
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.C2XHSIwH6D
deployment.apps/pxc-client created
+ cat /tmp/tmp.csyFaII3GM
+ rm /tmp/tmp.C2XHSIwH6D /tmp/tmp.csyFaII3GM
+ return 0
+ kubectl_bin apply -f -
+ yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"'
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.jN5FvcvQkz
++ mktemp
+ local LAST_ERR=/tmp/tmp.uPbQpGXkcl
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.jN5FvcvQkz
secret/minimal-cluster unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.uPbQpGXkcl
+ rm /tmp/tmp.jN5FvcvQkz /tmp/tmp.uPbQpGXkcl
+ return 0
+ yq eval '.spec.crVersion="9.9.9"'
+ yq eval '.spec.upgradeOptions.apply="8.4-latest"'
+ yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420"'
+ yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.4.5-5.1"'
+ yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"'
+ yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-28785.svc.cluster.local:11000"'
+ yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"'
+ kubectl_bin apply -f -
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cr-minimal.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZMnlzbcuBY
++ mktemp
+ local LAST_ERR=/tmp/tmp.lcdPbEyPAz
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ZMnlzbcuBY
perconaxtradbcluster.pxc.percona.com/minimal-cluster created
+ cat /tmp/tmp.lcdPbEyPAz
+ rm /tmp/tmp.ZMnlzbcuBY /tmp/tmp.lcdPbEyPAz
+ return 0
+ desc 'check if Pod is started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod is started
-----------------------------------------------------------------------------------

+ wait_for_running minimal-cluster-pxc 1
+ local name=minimal-cluster-pxc
+ local size=1
+ local last_pod
+ local max_retry=480
+ [[ -z minimal-cluster-pxc ]]
+ [[ 1 -le 0 ]]
+ last_pod=0
+ desc 'Waiting for 1 running pods for minimal-cluster-pxc'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 1 running pods for minimal-cluster-pxc
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in $(seq 0 $last_pod)
+ wait_pod minimal-cluster-pxc-0 480
+ local pod=minimal-cluster-pxc-0
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo minimal-cluster-pxc-0
+ local container=pxc
+ set +o xtrace
pod/minimal-cluster-pxc-0 condition met
waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok
+ sleep 20
+ local proxy
++ get_proxy minimal-cluster
++ local target_cluster=minimal-cluster
+++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0MedwhBoqY
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vxcXaVWZKE
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.0MedwhBoqY
+++ cat /tmp/tmp.vxcXaVWZKE
+++ rm /tmp/tmp.0MedwhBoqY /tmp/tmp.vxcXaVWZKE
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo minimal-cluster-haproxy
++ return
+ proxy=minimal-cluster-haproxy
+ wait_for_running minimal-cluster-haproxy 1
+ local name=minimal-cluster-haproxy
+ local size=1
+ local last_pod
+ local max_retry=480
+ [[ -z minimal-cluster-haproxy ]]
+ [[ 1 -le 0 ]]
+ last_pod=0
+ desc 'Waiting for 1 running pods for minimal-cluster-haproxy'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 1 running pods for minimal-cluster-haproxy
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in $(seq 0 $last_pod)
+ wait_pod minimal-cluster-haproxy-0 480
+ local pod=minimal-cluster-haproxy-0
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo minimal-cluster-haproxy-0
+ local container=
+ set +o xtrace
pod/minimal-cluster-haproxy-0 condition met
waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pHhOT1A3o3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ELHqjr1sMy
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pHhOT1A3o3
++ cat /tmp/tmp.ELHqjr1sMy
++ rm /tmp/tmp.pHhOT1A3o3 /tmp/tmp.ELHqjr1sMy
++ return 0
+ client_pod=pxc-client-6fc75f888d-4qprj
+ wait_pod pxc-client-6fc75f888d-4qprj
+ local pod=pxc-client-6fc75f888d-4qprj
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo pxc-client-6fc75f888d-4qprj
+ local container=
+ set +o xtrace
pod/pxc-client-6fc75f888d-4qprj condition met
waiting for pod/pxc-client-6fc75f888d-4qprj to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument.
command terminated with exit code 5
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PUceoo5HnX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1ge5swBa0j
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PUceoo5HnX
++ cat /tmp/tmp.1ge5swBa0j
++ rm /tmp/tmp.PUceoo5HnX /tmp/tmp.1ge5swBa0j
++ return 0
+ client_pod=pxc-client-6fc75f888d-4qprj
+ wait_pod pxc-client-6fc75f888d-4qprj
+ local pod=pxc-client-6fc75f888d-4qprj
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-6fc75f888d-4qprj
+ local container=
+ set +o xtrace
pod/pxc-client-6fc75f888d-4qprj condition met
waiting for pod/pxc-client-6fc75f888d-4qprj to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument.
command terminated with exit code 5
+ grep -E 'server request payload|unary call'
+ grep -Eo '\{.*\}'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ jq 'del(."grpc.time_ms")'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}'
+ kubectl_bin logs version-service-cr-58b8ffbbb4-wm2hj
++ mktemp
+ local LAST_OUT=/tmp/tmp.mM6mBFkiGr
++ mktemp
+ local LAST_ERR=/tmp/tmp.Daj9LhB0rd
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs version-service-cr-58b8ffbbb4-wm2hj
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.mM6mBFkiGr
+ cat /tmp/tmp.Daj9LhB0rd
+ rm /tmp/tmp.mM6mBFkiGr /tmp/tmp.Daj9LhB0rd
+ return 0
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ jq 'del(."grpc.time_ms")'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ grep -E 'server request payload|unary call'
+ grep -Eo '\{.*\}'
++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}'
+ kubectl_bin logs version-service-66774c865b-lzrlv
++ mktemp
+ local LAST_OUT=/tmp/tmp.XXw5ReNs0y
++ mktemp
+ local LAST_ERR=/tmp/tmp.kXHkjHN18z
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs version-service-66774c865b-lzrlv
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XXw5ReNs0y
+ cat /tmp/tmp.kXHkjHN18z
+ rm /tmp/tmp.XXw5ReNs0y /tmp/tmp.kXHkjHN18z
+ return 0
+ local telemetry_log_file=disabled_telemetry.version-service-cw.log.json
+ desc 'telemetry was disabled in CR but in operator not'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR but in operator not
-----------------------------------------------------------------------------------

+ '[' 8.4-latest == disabled -a disabled == enabled ']'
+ local image_prefix=8.4-latest
+ image_prefix=8.4
+ local telemetry_cr_log_file=disabled_telemetry.version-service-cr-8.4-cw.log.json
+ desc 'telemetry was disabled in operator but not in CR'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in operator but not in CR
-----------------------------------------------------------------------------------

+ [[ 8.4-latest == \8\.\4\-\r\e\c\o\m\m\e\n\d\e\d ]]
+ [[ 8.4-latest == \8\.\4\-\l\a\t\e\s\t ]]
+ '[' disabled == disabled ']'
+ desc 'cr VS should have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
cr VS should have telemetry
-----------------------------------------------------------------------------------

+ diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.4-cw.log.json /dev/fd/63
++ grep -f /tmp/tmp.kxnkOjofaU/disabled_telemetry.version-service-cr.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.4-cw.log.json
+ desc 'operator VS should not have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
operator VS should not have telemetry
-----------------------------------------------------------------------------------

+ [[ -s /tmp/tmp.kxnkOjofaU/disabled_telemetry.version-service.log.json ]]
+ desc 'telemetry was disabled in CR as well as in operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR as well as in operator
-----------------------------------------------------------------------------------

+ '[' 8.4-latest == disabled -a disabled == disabled ']'
+ kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.7hUMu43WFa
++ mktemp
+ local LAST_ERR=/tmp/tmp.1Tl50eV8Jk
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7hUMu43WFa
perconaxtradbcluster.pxc.percona.com/minimal-cluster patched
+ cat /tmp/tmp.1Tl50eV8Jk
+ rm /tmp/tmp.7hUMu43WFa /tmp/tmp.1Tl50eV8Jk
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ head -1
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BqbVlv5OPh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zyj10ngTMm
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BqbVlv5OPh
++ cat /tmp/tmp.zyj10ngTMm
++ rm /tmp/tmp.BqbVlv5OPh /tmp/tmp.zyj10ngTMm
++ return 0
+ kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-559d7d8944-b58tt
++ mktemp
+ local LAST_OUT=/tmp/tmp.5W6eSO2oPx
++ mktemp
+ local LAST_ERR=/tmp/tmp.cw3MEodVl5
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-559d7d8944-b58tt
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5W6eSO2oPx
pod "percona-xtradb-cluster-operator-559d7d8944-b58tt" deleted from pxc-operator namespace
+ cat /tmp/tmp.cw3MEodVl5
+ rm /tmp/tmp.5W6eSO2oPx /tmp/tmp.cw3MEodVl5
+ return 0
+ kubectl_bin delete pxc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.b8gl2O0jHO
++ mktemp
+ local LAST_ERR=/tmp/tmp.tsn22Fx9uu
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pxc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.b8gl2O0jHO
perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.tsn22Fx9uu
+ rm /tmp/tmp.b8gl2O0jHO /tmp/tmp.tsn22Fx9uu
+ return 0
+ kubectl_bin delete deploy pxc-client
++ mktemp
+ local LAST_OUT=/tmp/tmp.XF1f4JMjht
++ mktemp
+ local LAST_ERR=/tmp/tmp.h3RDfJLoH5
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete deploy pxc-client
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XF1f4JMjht
deployment.apps "pxc-client" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.h3RDfJLoH5
+ rm /tmp/tmp.XF1f4JMjht /tmp/tmp.h3RDfJLoH5
+ return 0
+ sleep 30
+ kubectl_bin delete pod -l run=version-service-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.rx6avajrvZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.xhHQoTz6dM
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pod -l run=version-service-cr
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.rx6avajrvZ
pod "version-service-cr-58b8ffbbb4-wm2hj" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.xhHQoTz6dM
+ rm /tmp/tmp.rx6avajrvZ /tmp/tmp.xhHQoTz6dM
+ return 0
+ kubectl_bin delete pod -l run=version-service
++ mktemp
+ local LAST_OUT=/tmp/tmp.CdgoC9o0uv
++ mktemp
+ local LAST_ERR=/tmp/tmp.nH3xbErn9X
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pod -l run=version-service
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.CdgoC9o0uv
pod "version-service-66774c865b-lzrlv" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.nH3xbErn9X
+ rm /tmp/tmp.CdgoC9o0uv /tmp/tmp.nH3xbErn9X
+ return 0
+ check_telemetry_transfer http://version-service-cr.smart-update2-28785.svc.cluster.local:11000 disabled disabled
+ local cr_vs_uri=http://version-service-cr.smart-update2-28785.svc.cluster.local:11000
+ local cr_vs_channel=disabled
+ local telemetry_state=disabled
+ desc 'create PXC minimal cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC minimal cluster
-----------------------------------------------------------------------------------

+ cluster=minimal-cluster
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.yGKbjl4XLn
++ mktemp
+ local LAST_ERR=/tmp/tmp.TJA4Fch9oX
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.yGKbjl4XLn
deployment.apps/pxc-client created
+ cat /tmp/tmp.TJA4Fch9oX
+ rm /tmp/tmp.yGKbjl4XLn /tmp/tmp.TJA4Fch9oX
+ return 0
+ yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"'
+ kubectl_bin apply -f -
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.OGq3tWdbPb
++ mktemp
+ local LAST_ERR=/tmp/tmp.6r2xZIMQCX
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.OGq3tWdbPb
secret/minimal-cluster unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.6r2xZIMQCX
+ rm /tmp/tmp.OGq3tWdbPb /tmp/tmp.6r2xZIMQCX
+ return 0
+ kubectl_bin apply -f -
+ yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"'
+ yq eval '.spec.crVersion="9.9.9"'
+ yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.4.5-5.1"'
+ yq eval '.spec.upgradeOptions.apply="disabled"'
+ yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"'
+ yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420"'
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cr-minimal.yaml
+ yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-28785.svc.cluster.local:11000"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.zxH4wIqrke
++ mktemp
+ local LAST_ERR=/tmp/tmp.3pOaj42Qp7
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.zxH4wIqrke
perconaxtradbcluster.pxc.percona.com/minimal-cluster created
+ cat /tmp/tmp.3pOaj42Qp7
+ rm /tmp/tmp.zxH4wIqrke /tmp/tmp.3pOaj42Qp7
+ return 0
+ desc 'check if Pod is started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod is started
-----------------------------------------------------------------------------------

+ wait_for_running minimal-cluster-pxc 1
+ local name=minimal-cluster-pxc
+ local size=1
+ local last_pod
+ local max_retry=480
+ [[ -z minimal-cluster-pxc ]]
+ [[ 1 -le 0 ]]
+ last_pod=0
+ desc 'Waiting for 1 running pods for minimal-cluster-pxc'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 1 running pods for minimal-cluster-pxc
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in $(seq 0 $last_pod)
+ wait_pod minimal-cluster-pxc-0 480
+ local pod=minimal-cluster-pxc-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/minimal-cluster-pxc-0 condition met
waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok
+ sleep 20
+ local proxy
++ get_proxy minimal-cluster
++ local target_cluster=minimal-cluster
+++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ipqLUugwdN
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.mS1s0Y1VPK
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.ipqLUugwdN
+++ cat /tmp/tmp.mS1s0Y1VPK
+++ rm /tmp/tmp.ipqLUugwdN /tmp/tmp.mS1s0Y1VPK
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo minimal-cluster-haproxy
++ return
+ proxy=minimal-cluster-haproxy
+ wait_for_running minimal-cluster-haproxy 1
+ local name=minimal-cluster-haproxy
+ local size=1
+ local last_pod
+ local max_retry=480
+ [[ -z minimal-cluster-haproxy ]]
+ [[ 1 -le 0 ]]
+ last_pod=0
+ desc 'Waiting for 1 running pods for minimal-cluster-haproxy'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 1 running pods for minimal-cluster-haproxy
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in $(seq 0 $last_pod)
+ wait_pod minimal-cluster-haproxy-0 480
+ local pod=minimal-cluster-haproxy-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/minimal-cluster-haproxy-0 condition met
waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DJgjD1xjS4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.maohICkXpJ
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DJgjD1xjS4
++ cat /tmp/tmp.maohICkXpJ
++ rm /tmp/tmp.DJgjD1xjS4 /tmp/tmp.maohICkXpJ
++ return 0
+ client_pod=pxc-client-6fc75f888d-62twb
+ wait_pod pxc-client-6fc75f888d-62twb
+ local pod=pxc-client-6fc75f888d-62twb
+ local max_retry=480
+ local ns=
++ echo pxc-client-6fc75f888d-62twb
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6fc75f888d-62twb condition met
waiting for pod/pxc-client-6fc75f888d-62twb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument.
command terminated with exit code 5
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9xOZRz16QA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oO2Ys0FdUZ
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.9xOZRz16QA
++ cat /tmp/tmp.oO2Ys0FdUZ
++ rm /tmp/tmp.9xOZRz16QA /tmp/tmp.oO2Ys0FdUZ
++ return 0
+ client_pod=pxc-client-6fc75f888d-62twb
+ wait_pod pxc-client-6fc75f888d-62twb
+ local pod=pxc-client-6fc75f888d-62twb
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ echo pxc-client-6fc75f888d-62twb
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6fc75f888d-62twb condition met
waiting for pod/pxc-client-6fc75f888d-62twb to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument.
command terminated with exit code 5
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ jq 'del(."grpc.time_ms")'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ grep -Eo '\{.*\}'
+ grep -E 'server request payload|unary call'
++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}'
+ kubectl_bin logs version-service-cr-58b8ffbbb4-5hs96
++ mktemp
+ local LAST_OUT=/tmp/tmp.mslWIqawbF
++ mktemp
+ local LAST_ERR=/tmp/tmp.bRUgAQ5phP
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs version-service-cr-58b8ffbbb4-5hs96
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.mslWIqawbF
+ cat /tmp/tmp.bRUgAQ5phP
+ rm /tmp/tmp.mslWIqawbF /tmp/tmp.bRUgAQ5phP
+ return 0
+ grep -Eo '\{.*\}'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ jq 'del(."grpc.time_ms")'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ grep -E 'server request payload|unary call'
++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}'
+ kubectl_bin logs version-service-66774c865b-wg5np
++ mktemp
+ local LAST_OUT=/tmp/tmp.SFYGkrtTIm
++ mktemp
+ local LAST_ERR=/tmp/tmp.1JzWGsv4FX
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs version-service-66774c865b-wg5np
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.SFYGkrtTIm
+ cat /tmp/tmp.1JzWGsv4FX
+ rm /tmp/tmp.SFYGkrtTIm /tmp/tmp.1JzWGsv4FX
+ return 0
+ local telemetry_log_file=disabled_telemetry.version-service-cw.log.json
+ desc 'telemetry was disabled in CR but in operator not'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR but in operator not
-----------------------------------------------------------------------------------

+ '[' disabled == disabled -a disabled == enabled ']'
+ local image_prefix=disabled
+ image_prefix=disabled
+ local telemetry_cr_log_file=disabled_telemetry.version-service-cr-disabled-cw.log.json
+ desc 'telemetry was disabled in operator but not in CR'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in operator but not in CR
-----------------------------------------------------------------------------------

+ [[ disabled == \d\i\s\a\b\l\e\d\-\r\e\c\o\m\m\e\n\d\e\d ]]
+ [[ disabled == \d\i\s\a\b\l\e\d\-\l\a\t\e\s\t ]]
+ desc 'telemetry was disabled in CR as well as in operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR as well as in operator
-----------------------------------------------------------------------------------

+ '[' disabled == disabled -a disabled == disabled ']'
+ desc 'CR VS should not have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
CR VS should not have telemetry
-----------------------------------------------------------------------------------

+ [[ -s /tmp/tmp.kxnkOjofaU/disabled_telemetry.version-service-cr.log.json ]]
+ desc 'operator VS should not have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
operator VS should not have telemetry
-----------------------------------------------------------------------------------

+ [[ -s /tmp/tmp.kxnkOjofaU/disabled_telemetry.version-service.log.json ]]
+ kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.EikIwJlYCj
++ mktemp
+ local LAST_ERR=/tmp/tmp.1qzQx28tLs
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EikIwJlYCj
perconaxtradbcluster.pxc.percona.com/minimal-cluster patched
+ cat /tmp/tmp.1qzQx28tLs
+ rm /tmp/tmp.EikIwJlYCj /tmp/tmp.1qzQx28tLs
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ head -1
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zPvon4oOXc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.peBYyGSHcX
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zPvon4oOXc
++ cat /tmp/tmp.peBYyGSHcX
++ rm /tmp/tmp.zPvon4oOXc /tmp/tmp.peBYyGSHcX
++ return 0
+ kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-559d7d8944-rhccw
++ mktemp
+ local LAST_OUT=/tmp/tmp.0lNvJFCv5C
++ mktemp
+ local LAST_ERR=/tmp/tmp.HnAQhgd8a8
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-559d7d8944-rhccw
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.0lNvJFCv5C
pod "percona-xtradb-cluster-operator-559d7d8944-rhccw" deleted from pxc-operator namespace
+ cat /tmp/tmp.HnAQhgd8a8
+ rm /tmp/tmp.0lNvJFCv5C /tmp/tmp.HnAQhgd8a8
+ return 0
+ kubectl_bin delete pxc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.HEMRwoQ2BO
++ mktemp
+ local LAST_ERR=/tmp/tmp.er5iHld3ok
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pxc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.HEMRwoQ2BO
perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.er5iHld3ok
+ rm /tmp/tmp.HEMRwoQ2BO /tmp/tmp.er5iHld3ok
+ return 0
+ kubectl_bin delete deploy pxc-client
++ mktemp
+ local LAST_OUT=/tmp/tmp.MgNODlh9JH
++ mktemp
+ local LAST_ERR=/tmp/tmp.3Co0o4h8YA
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete deploy pxc-client
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MgNODlh9JH
deployment.apps "pxc-client" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.3Co0o4h8YA
+ rm /tmp/tmp.MgNODlh9JH /tmp/tmp.3Co0o4h8YA
+ return 0
+ sleep 30
+ kubectl_bin delete deployment version-service-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.g1z3cGH8bS
++ mktemp
+ local LAST_ERR=/tmp/tmp.lpGqFE5xDq
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete deployment version-service-cr
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.g1z3cGH8bS
deployment.apps "version-service-cr" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.lpGqFE5xDq
+ rm /tmp/tmp.g1z3cGH8bS /tmp/tmp.lpGqFE5xDq
+ return 0
+ desc 'Telemetry testing finished'
+ set +o xtrace


-----------------------------------------------------------------------------------
Telemetry testing finished
-----------------------------------------------------------------------------------

+ desc 'PXC cluster with version service offline'
+ set +o xtrace


-----------------------------------------------------------------------------------
PXC cluster with version service offline
-----------------------------------------------------------------------------------

+ cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/smart-update-version-service-unreachable.yml /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420"' /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ spinup_pxc smart-update /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local cluster=smart-update
+ local config=/tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.hYL4rZxWJ0
++ mktemp
+ local LAST_ERR=/tmp/tmp.myFOFqzbMf
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.hYL4rZxWJ0
secret/my-cluster-secrets created
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.myFOFqzbMf
+ rm /tmp/tmp.hYL4rZxWJ0 /tmp/tmp.myFOFqzbMf
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ local pvc_name=
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-28785~
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.QW7NWHURqp
++ mktemp
+ local LAST_ERR=/tmp/tmp.E6zpzPDVUi
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QW7NWHURqp
deployment.apps/pxc-client created
+ cat /tmp/tmp.E6zpzPDVUi
+ rm /tmp/tmp.QW7NWHURqp /tmp/tmp.E6zpzPDVUi
+ return 0
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ apply_config /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local config_file=/tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local pvc_name=
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml ''
+ local input_file=/tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local pvc_name=
++ mktemp
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#'
+ cat /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-28785~
+ local LAST_OUT=/tmp/tmp.co0kRAuUsy
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
++ mktemp
+ local LAST_ERR=/tmp/tmp.cmnGAwqX10
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.co0kRAuUsy
perconaxtradbcluster.pxc.percona.com/smart-update created
+ cat /tmp/tmp.cmnGAwqX10
+ rm /tmp/tmp.co0kRAuUsy /tmp/tmp.cmnGAwqX10
+ return 0
+ desc 'waiting pods to be running for smart-update'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting pods to be running for smart-update
-----------------------------------------------------------------------------------

++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.NpllwBckyh
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cHXQynxkhp
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.NpllwBckyh
+++ cat /tmp/tmp.cHXQynxkhp
+++ rm /tmp/tmp.NpllwBckyh /tmp/tmp.cHXQynxkhp
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ local proxy=smart-update-haproxy
++ get_proxy_size smart-update
++ local cluster=smart-update
++ local size=0
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.K7bCzjOBfr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vyi1gWKzgw
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.K7bCzjOBfr
+++ cat /tmp/tmp.vyi1gWKzgw
+++ rm /tmp/tmp.K7bCzjOBfr /tmp/tmp.vyi1gWKzgw
+++ return 0
++ [[ true == \t\r\u\e ]]
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.NWS1kHdKUS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.SoCJA75wso
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.NWS1kHdKUS
+++ cat /tmp/tmp.SoCJA75wso
+++ rm /tmp/tmp.NWS1kHdKUS /tmp/tmp.SoCJA75wso
+++ return 0
++ size=2
++ echo 2
+ local proxy_size=2
+ wait_for_running smart-update-pxc 3
+ local name=smart-update-pxc
+ local size=3
+ local last_pod
+ local max_retry=480
+ [[ -z smart-update-pxc ]]
+ [[ 3 -le 0 ]]
+ last_pod=2
+ desc 'Waiting for 3 running pods for smart-update-pxc'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 3 running pods for smart-update-pxc
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-pxc-0 480
+ local pod=smart-update-pxc-0
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo smart-update-pxc-0
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-0 condition met
waiting for pod/smart-update-pxc-0 to become Ready.Ok
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-pxc-1 480
+ local pod=smart-update-pxc-1
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo smart-update-pxc-1
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-1 condition met
waiting for pod/smart-update-pxc-1 to become Ready.Ok
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-pxc-2 480
+ local pod=smart-update-pxc-2
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo smart-update-pxc-2
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-2 condition met
waiting for pod/smart-update-pxc-2 to become Ready.Ok
+ wait_for_running smart-update-haproxy 2
+ local name=smart-update-haproxy
+ local size=2
+ local last_pod
+ local max_retry=480
+ [[ -z smart-update-haproxy ]]
+ [[ 2 -le 0 ]]
+ last_pod=1
+ desc 'Waiting for 2 running pods for smart-update-haproxy'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 2 running pods for smart-update-haproxy
-----------------------------------------------------------------------------------

++ seq 0 1
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-haproxy-0 480
+ local pod=smart-update-haproxy-0
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo smart-update-haproxy-0
+ local container=
+ set +o xtrace
pod/smart-update-haproxy-0 condition met
waiting for pod/smart-update-haproxy-0 to become Ready.Ok
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-haproxy-1 480
+ local pod=smart-update-haproxy-1
+ local max_retry=480
+ local ns=
++ echo smart-update-haproxy-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/smart-update-haproxy-1 condition met
waiting for pod/smart-update-haproxy-1 to become Ready.Ok
+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local pxc_size=3
+ local proxy_size=2
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local max_seconds=1500
+ local delay=5
+ local retries=300
+ local pxc_ready
+ local proxy_ready
+ sleep 7
+ echo -n 'waiting for pxc/smart-update to be ready'
waiting for pxc/smart-update to be ready++ seq 1 300
+ for _ in $(seq 1 "${retries}")
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cSao7J8c5y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SOulSLnYYd
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.cSao7J8c5y
++ cat /tmp/tmp.SOulSLnYYd
++ rm /tmp/tmp.cSao7J8c5y /tmp/tmp.SOulSLnYYd
++ return 0
+ state=ready
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7vxvLYwOhl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5yU9vzwkqa
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7vxvLYwOhl
++ cat /tmp/tmp.5yU9vzwkqa
++ rm /tmp/tmp.7vxvLYwOhl /tmp/tmp.5yU9vzwkqa
++ return 0
+ pxc_ready=3
+ proxy_ready=0
+ [[ 2 -gt 0 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.84WSDEQGVI
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.YTsY2tRACD
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.84WSDEQGVI
+++++ cat /tmp/tmp.YTsY2tRACD
+++++ rm /tmp/tmp.84WSDEQGVI /tmp/tmp.YTsY2tRACD
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sQOi5LrnJM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XaDAzhUXBZ
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sQOi5LrnJM
++ cat /tmp/tmp.XaDAzhUXBZ
++ rm /tmp/tmp.sQOi5LrnJM /tmp/tmp.XaDAzhUXBZ
++ return 0
+ proxy_ready=2
+ [[ ready == \r\e\a\d\y ]]
+ [[ 3 == \3 ]]
+ [[ 2 == \2 ]]
+ echo

+ return
+ sleep 10
++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zRfdpvpVaL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Qc8a1tEjPz
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zRfdpvpVaL
++ cat /tmp/tmp.Qc8a1tEjPz
++ rm /tmp/tmp.zRfdpvpVaL /tmp/tmp.Qc8a1tEjPz
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AUx7D1l8Z5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XXeHJlsh99
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AUx7D1l8Z5
++ cat /tmp/tmp.XXeHJlsh99
++ rm /tmp/tmp.AUx7D1l8Z5 /tmp/tmp.XXeHJlsh99
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-674d6c4f45-9z9z4
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ write_data smart-update 100500 no-proxy -P3306
+ local cluster=smart-update
+ local id=100500
+ local use_proxy=no-proxy
+ local port=-P3306
+ local database=myApp
+ local table=myApp
+ local root_pass
++ get_password smart-update root
++ local cluster=smart-update
++ local user=root
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.rS8rIOOVxK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NEMzLyZBWw
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.rS8rIOOVxK
+++ cat /tmp/tmp.NEMzLyZBWw
+++ rm /tmp/tmp.rS8rIOOVxK /tmp/tmp.NEMzLyZBWw
+++ return 0
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ base64 --decode
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UgUBfMuKCk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TC59rsTflc
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.UgUBfMuKCk
++ cat /tmp/tmp.TC59rsTflc
++ rm /tmp/tmp.UgUBfMuKCk /tmp/tmp.TC59rsTflc
++ return 0
+ root_pass=root_password
++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.T2hRRtH1xc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kYs6FpADUg
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.T2hRRtH1xc
+++ cat /tmp/tmp.kYs6FpADUg
+++ rm /tmp/tmp.T2hRRtH1xc /tmp/tmp.kYs6FpADUg
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ proxy=smart-update-haproxy
+ [[ no-proxy == true ]]
+ [[ no-proxy == \u\s\e\-\p\r\o\x\y ]]
+ host='-h smart-update-pxc-0.smart-update-pxc'
+ run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3EnQ8qpGfO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1Dqv1tb0wU
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3EnQ8qpGfO
++ cat /tmp/tmp.1Dqv1tb0wU
++ rm /tmp/tmp.3EnQ8qpGfO /tmp/tmp.1Dqv1tb0wU
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo pxc-client-674d6c4f45-9z9z4
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
+ desc 'compare initial data'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare initial data
-----------------------------------------------------------------------------------

+ compare_data smart-update select-1 no-proxy -P3306
+ local cluster=smart-update
+ local compare=select-1
+ local use_proxy=no-proxy
+ local port=-P3306
+ local database=myApp
+ local table=myApp
+ local size
+ local root_pass
++ get_password smart-update root
++ local cluster=smart-update
++ local user=root
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.VKsruTGfVS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.GmDRbAEGNW
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.VKsruTGfVS
+++ cat /tmp/tmp.GmDRbAEGNW
+++ rm /tmp/tmp.VKsruTGfVS /tmp/tmp.GmDRbAEGNW
+++ return 0
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ base64 --decode
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EP3YpVXDOr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pbCAq9ovj7
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.EP3YpVXDOr
++ cat /tmp/tmp.pbCAq9ovj7
++ rm /tmp/tmp.EP3YpVXDOr /tmp/tmp.pbCAq9ovj7
++ return 0
+ root_pass=root_password
+ [[ no-proxy == true ]]
+ [[ no-proxy == \u\s\e\-\p\r\o\x\y ]]
++ get_pxc_size smart-update
++ local cluster=smart-update
++ local size=0
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.NgD8n94zxE
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.uObBL2wjLB
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.NgD8n94zxE
+++ cat /tmp/tmp.uObBL2wjLB
+++ rm /tmp/tmp.NgD8n94zxE /tmp/tmp.uObBL2wjLB
+++ return 0
++ size=3
++ echo 3
+ size=3
++ seq 0 2
+ for i in $(seq 0 $((size - 1)))
+ compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local command_id=select-1
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Y9dUxeZmTn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NKd8rMwHyA
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Y9dUxeZmTn
++ cat /tmp/tmp.NKd8rMwHyA
++ rm /tmp/tmp.Y9dUxeZmTn /tmp/tmp.NKd8rMwHyA
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-674d6c4f45-9z9z4
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ for i in $(seq 0 $((size - 1)))
+ compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local command_id=select-1
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WLdp8wFUxN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2Ro8UXrzdD
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WLdp8wFUxN
++ cat /tmp/tmp.2Ro8UXrzdD
++ rm /tmp/tmp.WLdp8wFUxN /tmp/tmp.2Ro8UXrzdD
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-674d6c4f45-9z9z4
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ for i in $(seq 0 $((size - 1)))
+ compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local command_id=select-1
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.o9QQhZPwFZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OjQv5wySqG
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.o9QQhZPwFZ
++ cat /tmp/tmp.OjQv5wySqG
++ rm /tmp/tmp.o9QQhZPwFZ /tmp/tmp.OjQv5wySqG
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-674d6c4f45-9z9z4
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ is_keyring_plugin_in_use smart-update
+ local cluster=smart-update
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ kubectl exec smart-update-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my
+ return 1
+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local pxc_size=3
+ local proxy_size=2
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local max_seconds=1500
+ local delay=5
+ local retries=300
+ local pxc_ready
+ local proxy_ready
+ sleep 7
+ echo -n 'waiting for pxc/smart-update to be ready'
waiting for pxc/smart-update to be ready++ seq 1 300
+ for _ in $(seq 1 "${retries}")
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6EgObr8pB6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MwaxWcV5re
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6EgObr8pB6
++ cat /tmp/tmp.MwaxWcV5re
++ rm /tmp/tmp.6EgObr8pB6 /tmp/tmp.MwaxWcV5re
++ return 0
+ state=ready
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5ZBzeYrB2D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DCck4KrQKd
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.5ZBzeYrB2D
++ cat /tmp/tmp.DCck4KrQKd
++ rm /tmp/tmp.5ZBzeYrB2D /tmp/tmp.DCck4KrQKd
++ return 0
+ pxc_ready=3
+ proxy_ready=0
+ [[ 2 -gt 0 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.BkZILSSND0
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.hdfoNM9efe
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.BkZILSSND0
+++++ cat /tmp/tmp.hdfoNM9efe
+++++ rm /tmp/tmp.BkZILSSND0 /tmp/tmp.hdfoNM9efe
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rt74dn9a7N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qvPvgbKnLC
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rt74dn9a7N
++ cat /tmp/tmp.qvPvgbKnLC
++ rm /tmp/tmp.rt74dn9a7N /tmp/tmp.qvPvgbKnLC
++ return 0
+ proxy_ready=2
+ [[ ready == \r\e\a\d\y ]]
+ [[ 3 == \3 ]]
+ [[ 2 == \2 ]]
+ echo

+ return
++ kubectl_bin get pxc/smart-update -o 'jsonpath={.spec.pxc.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Az7mDOAAkJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gCclzKINan
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc/smart-update -o 'jsonpath={.spec.pxc.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Az7mDOAAkJ
++ cat /tmp/tmp.gCclzKINan
++ rm /tmp/tmp.Az7mDOAAkJ /tmp/tmp.gCclzKINan
++ return 0
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 != \p\e\r\c\o\n\a\/\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\:\8\.\4\.\5\-\5\.\1 ]]
+ desc 'PXC cluster update with latest image by version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
PXC cluster update with latest image by version service
-----------------------------------------------------------------------------------

+ vs_image=latest
++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password'
++ local 'command=SELECT @@hostname hostname;'
++ local 'uri=-h smart-update-haproxy -uroot -proot_password'
+++ get_client_pod
+++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.c9k7yGxwA9
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.RgS06xMSll
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.c9k7yGxwA9
+++ cat /tmp/tmp.RgS06xMSll
+++ rm /tmp/tmp.c9k7yGxwA9 /tmp/tmp.RgS06xMSll
+++ return 0
++ client_pod=pxc-client-674d6c4f45-9z9z4
++ wait_pod pxc-client-674d6c4f45-9z9z4
++ local pod=pxc-client-674d6c4f45-9z9z4
++ local max_retry=480
++ local ns=
+++ grep -E '^(pxc|proxysql)$'
+++ echo pxc-client-674d6c4f45-9z9z4
+++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ local container=
++ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
++ set +o xtrace
+ initial_primary=smart-update-pxc-0
+ kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.4P3CDR59cj
++ mktemp
+ local LAST_ERR=/tmp/tmp.J4aak6FW6i
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.4P3CDR59cj
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ cat /tmp/tmp.J4aak6FW6i
+ rm /tmp/tmp.4P3CDR59cj /tmp/tmp.J4aak6FW6i
+ return 0
+ sleep 55
+ check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.4
+ local cluster=smart-update
+ local initial_primary=smart-update-pxc-0
+ local pxc_size=3
+ local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.4
+ set +x
Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local pxc_size=3
+ local proxy_size=2
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local max_seconds=1500
+ local delay=5
+ local retries=300
+ local pxc_ready
+ local proxy_ready
+ sleep 7
+ echo -n 'waiting for pxc/smart-update to be ready'
waiting for pxc/smart-update to be ready++ seq 1 300
+ for _ in $(seq 1 "${retries}")
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ueSM1KNMvx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.t1R0Ipddzz
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ueSM1KNMvx
++ cat /tmp/tmp.t1R0Ipddzz
++ rm /tmp/tmp.ueSM1KNMvx /tmp/tmp.t1R0Ipddzz
++ return 0
+ state=ready
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OqyQg0cPmF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OMYQtXRORT
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.OqyQg0cPmF
++ cat /tmp/tmp.OMYQtXRORT
++ rm /tmp/tmp.OqyQg0cPmF /tmp/tmp.OMYQtXRORT
++ return 0
+ pxc_ready=3
+ proxy_ready=0
+ [[ 2 -gt 0 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.kMgGbD0swt
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.W557vijgRp
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.kMgGbD0swt
+++++ cat /tmp/tmp.W557vijgRp
+++++ rm /tmp/tmp.kMgGbD0swt /tmp/tmp.W557vijgRp
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MhdTko5Pvo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jnHj5jZiN2
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.MhdTko5Pvo
++ cat /tmp/tmp.jnHj5jZiN2
++ rm /tmp/tmp.MhdTko5Pvo /tmp/tmp.jnHj5jZiN2
++ return 0
+ proxy_ready=2
+ [[ ready == \r\e\a\d\y ]]
+ [[ 3 == \3 ]]
+ [[ 2 == \2 ]]
+ echo

+ return
++ seq 0 2
+ for i in $(seq 0 $((CLUSTER_SIZE - 1)))
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ysdfCkrefs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.W4abbHBlXW
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ysdfCkrefs
++ cat /tmp/tmp.W4abbHBlXW
++ rm /tmp/tmp.ysdfCkrefs /tmp/tmp.W4abbHBlXW
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ echo pxc-client-674d6c4f45-9z9z4
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ for i in $(seq 0 $((CLUSTER_SIZE - 1)))
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eveeILb3cR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LzVEyKcLTB
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eveeILb3cR
++ cat /tmp/tmp.LzVEyKcLTB
++ rm /tmp/tmp.eveeILb3cR /tmp/tmp.LzVEyKcLTB
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ echo pxc-client-674d6c4f45-9z9z4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ for i in $(seq 0 $((CLUSTER_SIZE - 1)))
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qva78ba26I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fS5YI1RrsM
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.qva78ba26I
++ cat /tmp/tmp.fS5YI1RrsM
++ rm /tmp/tmp.qva78ba26I /tmp/tmp.fS5YI1RrsM
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-674d6c4f45-9z9z4
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ kubectl_bin delete -f /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.lXzz74PrHg
++ mktemp
+ local LAST_ERR=/tmp/tmp.c7YgLOJjhP
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lXzz74PrHg
perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.c7YgLOJjhP
+ rm /tmp/tmp.lXzz74PrHg /tmp/tmp.c7YgLOJjhP
+ return 0
+ kubectl_bin delete pvc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.ff7HjjZt35
++ mktemp
+ local LAST_ERR=/tmp/tmp.g9UkvdTwoe
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pvc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ff7HjjZt35
persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-28785 namespace
persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-28785 namespace
persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.g9UkvdTwoe
+ rm /tmp/tmp.ff7HjjZt35 /tmp/tmp.g9UkvdTwoe
+ return 0
+ desc 'PXC cluster update with the latest image by version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
PXC cluster update with the latest image by version service
-----------------------------------------------------------------------------------

+ spinup_pxc smart-update /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local cluster=smart-update
+ local config=/tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.UDi6ERcaKM
++ mktemp
+ local LAST_ERR=/tmp/tmp.LVAHxc4x7p
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UDi6ERcaKM
secret/my-cluster-secrets unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.LVAHxc4x7p
+ rm /tmp/tmp.UDi6ERcaKM /tmp/tmp.LVAHxc4x7p
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ local pvc_name=
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.cbkVP4GJt3
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-28785~
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ local LAST_ERR=/tmp/tmp.zbhEFdvVqd
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.cbkVP4GJt3
deployment.apps/pxc-client unchanged
+ cat /tmp/tmp.zbhEFdvVqd
+ rm /tmp/tmp.cbkVP4GJt3 /tmp/tmp.zbhEFdvVqd
+ return 0
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ apply_config /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local config_file=/tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml ''
+ local input_file=/tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local pvc_name=
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.uvLsAjKA0i
++ mktemp
+ local LAST_ERR=/tmp/tmp.JtFbQjBjyq
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#'
+ cat /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-28785~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.uvLsAjKA0i
perconaxtradbcluster.pxc.percona.com/smart-update created
+ cat /tmp/tmp.JtFbQjBjyq
+ rm /tmp/tmp.uvLsAjKA0i /tmp/tmp.JtFbQjBjyq
+ return 0
+ desc 'waiting pods to be running for smart-update'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting pods to be running for smart-update
-----------------------------------------------------------------------------------

++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.snEAPfKo84
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.lv5Kcygbmt
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.snEAPfKo84
+++ cat /tmp/tmp.lv5Kcygbmt
+++ rm /tmp/tmp.snEAPfKo84 /tmp/tmp.lv5Kcygbmt
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ local proxy=smart-update-haproxy
++ get_proxy_size smart-update
++ local cluster=smart-update
++ local size=0
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zd6Qb7OZUB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.whE3NgW6Ci
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.zd6Qb7OZUB
+++ cat /tmp/tmp.whE3NgW6Ci
+++ rm /tmp/tmp.zd6Qb7OZUB /tmp/tmp.whE3NgW6Ci
+++ return 0
++ [[ true == \t\r\u\e ]]
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.hsIRsrVJ5Z
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VD0hMLjoUU
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.hsIRsrVJ5Z
+++ cat /tmp/tmp.VD0hMLjoUU
+++ rm /tmp/tmp.hsIRsrVJ5Z /tmp/tmp.VD0hMLjoUU
+++ return 0
++ size=2
++ echo 2
+ local proxy_size=2
+ wait_for_running smart-update-pxc 3
+ local name=smart-update-pxc
+ local size=3
+ local last_pod
+ local max_retry=480
+ [[ -z smart-update-pxc ]]
+ [[ 3 -le 0 ]]
+ last_pod=2
+ desc 'Waiting for 3 running pods for smart-update-pxc'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 3 running pods for smart-update-pxc
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-pxc-0 480
+ local pod=smart-update-pxc-0
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-0 condition met
waiting for pod/smart-update-pxc-0 to become Ready.Ok
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-pxc-1 480
+ local pod=smart-update-pxc-1
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ echo smart-update-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-1 condition met
waiting for pod/smart-update-pxc-1 to become Ready.Ok
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-pxc-2 480
+ local pod=smart-update-pxc-2
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo smart-update-pxc-2
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-2 condition met
waiting for pod/smart-update-pxc-2 to become Ready.Ok
+ wait_for_running smart-update-haproxy 2
+ local name=smart-update-haproxy
+ local size=2
+ local last_pod
+ local max_retry=480
+ [[ -z smart-update-haproxy ]]
+ [[ 2 -le 0 ]]
+ last_pod=1
+ desc 'Waiting for 2 running pods for smart-update-haproxy'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 2 running pods for smart-update-haproxy
-----------------------------------------------------------------------------------

++ seq 0 1
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-haproxy-0 480
+ local pod=smart-update-haproxy-0
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo smart-update-haproxy-0
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/smart-update-haproxy-0 condition met
waiting for pod/smart-update-haproxy-0 to become Ready.Ok
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-haproxy-1 480
+ local pod=smart-update-haproxy-1
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo smart-update-haproxy-1
+ local container=
+ set +o xtrace
pod/smart-update-haproxy-1 condition met
waiting for pod/smart-update-haproxy-1 to become Ready.Ok
+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local pxc_size=3
+ local proxy_size=2
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local max_seconds=1500
+ local delay=5
+ local retries=300
+ local pxc_ready
+ local proxy_ready
+ sleep 7
+ echo -n 'waiting for pxc/smart-update to be ready'
waiting for pxc/smart-update to be ready++ seq 1 300
+ for _ in $(seq 1 "${retries}")
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.druKnCwt2J
+++ mktemp
++ local LAST_ERR=/tmp/tmp.n4GGggu9Jh
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.druKnCwt2J
++ cat /tmp/tmp.n4GGggu9Jh
++ rm /tmp/tmp.druKnCwt2J /tmp/tmp.n4GGggu9Jh
++ return 0
+ state=ready
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.apOMUr4KeO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.evesmjhA9H
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.apOMUr4KeO
++ cat /tmp/tmp.evesmjhA9H
++ rm /tmp/tmp.apOMUr4KeO /tmp/tmp.evesmjhA9H
++ return 0
+ pxc_ready=3
+ proxy_ready=0
+ [[ 2 -gt 0 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.aQ0V9aE4pu
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.zKQpM8DJvW
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.aQ0V9aE4pu
+++++ cat /tmp/tmp.zKQpM8DJvW
+++++ rm /tmp/tmp.aQ0V9aE4pu /tmp/tmp.zKQpM8DJvW
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pNOVGVqyTY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5MzcKkrobx
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pNOVGVqyTY
++ cat /tmp/tmp.5MzcKkrobx
++ rm /tmp/tmp.pNOVGVqyTY /tmp/tmp.5MzcKkrobx
++ return 0
+ proxy_ready=2
+ [[ ready == \r\e\a\d\y ]]
+ [[ 3 == \3 ]]
+ [[ 2 == \2 ]]
+ echo

+ return
+ sleep 10
++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ base64 --decode
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.id4rOKVYQV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qwSx3wfo4k
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.id4rOKVYQV
++ cat /tmp/tmp.qwSx3wfo4k
++ rm /tmp/tmp.id4rOKVYQV /tmp/tmp.qwSx3wfo4k
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PUUfo7wrSB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Apctnobk9E
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PUUfo7wrSB
++ cat /tmp/tmp.Apctnobk9E
++ rm /tmp/tmp.PUUfo7wrSB /tmp/tmp.Apctnobk9E
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ echo pxc-client-674d6c4f45-9z9z4
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ write_data smart-update 100500 no-proxy -P3306
+ local cluster=smart-update
+ local id=100500
+ local use_proxy=no-proxy
+ local port=-P3306
+ local database=myApp
+ local table=myApp
+ local root_pass
++ get_password smart-update root
++ local cluster=smart-update
++ local user=root
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.sYA9wSsdm0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IUUh2nf9gY
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.sYA9wSsdm0
+++ cat /tmp/tmp.IUUh2nf9gY
+++ rm /tmp/tmp.sYA9wSsdm0 /tmp/tmp.IUUh2nf9gY
+++ return 0
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ base64 --decode
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xngBUasVLz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0zTCdnnkNd
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xngBUasVLz
++ cat /tmp/tmp.0zTCdnnkNd
++ rm /tmp/tmp.xngBUasVLz /tmp/tmp.0zTCdnnkNd
++ return 0
+ root_pass=root_password
++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.X1ElQUPt8p
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bu1IGybuZ5
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.X1ElQUPt8p
+++ cat /tmp/tmp.bu1IGybuZ5
+++ rm /tmp/tmp.X1ElQUPt8p /tmp/tmp.bu1IGybuZ5
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ proxy=smart-update-haproxy
+ [[ no-proxy == true ]]
+ [[ no-proxy == \u\s\e\-\p\r\o\x\y ]]
+ host='-h smart-update-pxc-0.smart-update-pxc'
+ run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RQk6uvLwxT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.htfx9A5GWj
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RQk6uvLwxT
++ cat /tmp/tmp.htfx9A5GWj
++ rm /tmp/tmp.RQk6uvLwxT /tmp/tmp.htfx9A5GWj
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ echo pxc-client-674d6c4f45-9z9z4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
+ desc 'compare initial data'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare initial data
-----------------------------------------------------------------------------------

+ compare_data smart-update select-1 no-proxy -P3306
+ local cluster=smart-update
+ local compare=select-1
+ local use_proxy=no-proxy
+ local port=-P3306
+ local database=myApp
+ local table=myApp
+ local size
+ local root_pass
++ get_password smart-update root
++ local cluster=smart-update
++ local user=root
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SSiBMj6a9s
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VjoFL98TXY
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.SSiBMj6a9s
+++ cat /tmp/tmp.VjoFL98TXY
+++ rm /tmp/tmp.SSiBMj6a9s /tmp/tmp.VjoFL98TXY
+++ return 0
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1uHXoToyk5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.f3gZw2bRqb
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1uHXoToyk5
++ cat /tmp/tmp.f3gZw2bRqb
++ rm /tmp/tmp.1uHXoToyk5 /tmp/tmp.f3gZw2bRqb
++ return 0
+ root_pass=root_password
+ [[ no-proxy == true ]]
+ [[ no-proxy == \u\s\e\-\p\r\o\x\y ]]
++ get_pxc_size smart-update
++ local cluster=smart-update
++ local size=0
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.GEgNfpI2tz
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1mTIvzul6J
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.GEgNfpI2tz
+++ cat /tmp/tmp.1mTIvzul6J
+++ rm /tmp/tmp.GEgNfpI2tz /tmp/tmp.1mTIvzul6J
+++ return 0
++ size=3
++ echo 3
+ size=3
++ seq 0 2
+ for i in $(seq 0 $((size - 1)))
+ compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local command_id=select-1
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BnJ5ILO8Mp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8ePG4SOebf
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BnJ5ILO8Mp
++ cat /tmp/tmp.8ePG4SOebf
++ rm /tmp/tmp.BnJ5ILO8Mp /tmp/tmp.8ePG4SOebf
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-674d6c4f45-9z9z4
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ for i in $(seq 0 $((size - 1)))
+ compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local command_id=select-1
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KHmhj3xdrt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3g2Noh73hW
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KHmhj3xdrt
++ cat /tmp/tmp.3g2Noh73hW
++ rm /tmp/tmp.KHmhj3xdrt /tmp/tmp.3g2Noh73hW
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo pxc-client-674d6c4f45-9z9z4
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ for i in $(seq 0 $((size - 1)))
+ compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local command_id=select-1
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.f3wQLDmeaR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SmsKYSgdlD
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.f3wQLDmeaR
++ cat /tmp/tmp.SmsKYSgdlD
++ rm /tmp/tmp.f3wQLDmeaR /tmp/tmp.SmsKYSgdlD
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-674d6c4f45-9z9z4
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ is_keyring_plugin_in_use smart-update
+ local cluster=smart-update
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ kubectl exec smart-update-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my
+ return 1
+ vs_image=latest
++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password'
++ local 'command=SELECT @@hostname hostname;'
++ local 'uri=-h smart-update-haproxy -uroot -proot_password'
+++ get_client_pod
+++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tzlovlLEDj
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.El5Z7Iu4Vm
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.tzlovlLEDj
+++ cat /tmp/tmp.El5Z7Iu4Vm
+++ rm /tmp/tmp.tzlovlLEDj /tmp/tmp.El5Z7Iu4Vm
+++ return 0
++ client_pod=pxc-client-674d6c4f45-9z9z4
++ wait_pod pxc-client-674d6c4f45-9z9z4
++ local pod=pxc-client-674d6c4f45-9z9z4
++ local max_retry=480
++ local ns=
+++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+++ grep -E '^(pxc|proxysql)$'
+++ echo pxc-client-674d6c4f45-9z9z4
++ local container=
++ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
++ set +o xtrace
+ initial_primary=smart-update-pxc-0
+ kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.REwXvaTllZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.awvo8N7zTp
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.REwXvaTllZ
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ cat /tmp/tmp.awvo8N7zTp
+ rm /tmp/tmp.REwXvaTllZ /tmp/tmp.awvo8N7zTp
+ return 0
+ sleep 55
+ check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.4
+ local cluster=smart-update
+ local initial_primary=smart-update-pxc-0
+ local pxc_size=3
+ local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.4
+ set +x
Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local pxc_size=3
+ local proxy_size=2
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local max_seconds=1500
+ local delay=5
+ local retries=300
+ local pxc_ready
+ local proxy_ready
+ sleep 7
+ echo -n 'waiting for pxc/smart-update to be ready'
waiting for pxc/smart-update to be ready++ seq 1 300
+ for _ in $(seq 1 "${retries}")
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bxvNrjssRK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v3gr5FrR62
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bxvNrjssRK
++ cat /tmp/tmp.v3gr5FrR62
++ rm /tmp/tmp.bxvNrjssRK /tmp/tmp.v3gr5FrR62
++ return 0
+ state=ready
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kIp9jQZYUV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FCGbmVKMyJ
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.kIp9jQZYUV
++ cat /tmp/tmp.FCGbmVKMyJ
++ rm /tmp/tmp.kIp9jQZYUV /tmp/tmp.FCGbmVKMyJ
++ return 0
+ pxc_ready=3
+ proxy_ready=0
+ [[ 2 -gt 0 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.4uihViOHUo
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.wx2Gykm7OU
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.4uihViOHUo
+++++ cat /tmp/tmp.wx2Gykm7OU
+++++ rm /tmp/tmp.4uihViOHUo /tmp/tmp.wx2Gykm7OU
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UZqh2a9dEj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TXvaU9f7qU
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.UZqh2a9dEj
++ cat /tmp/tmp.TXvaU9f7qU
++ rm /tmp/tmp.UZqh2a9dEj /tmp/tmp.TXvaU9f7qU
++ return 0
+ proxy_ready=2
+ [[ ready == \r\e\a\d\y ]]
+ [[ 3 == \3 ]]
+ [[ 2 == \2 ]]
+ echo

+ return
++ seq 0 2
+ for i in $(seq 0 $((CLUSTER_SIZE - 1)))
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3QRmvj4VGT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ulGmJn5XuK
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3QRmvj4VGT
++ cat /tmp/tmp.ulGmJn5XuK
++ rm /tmp/tmp.3QRmvj4VGT /tmp/tmp.ulGmJn5XuK
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ echo pxc-client-674d6c4f45-9z9z4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ for i in $(seq 0 $((CLUSTER_SIZE - 1)))
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sqMbTZYZwv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.26Cqly6JCm
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sqMbTZYZwv
++ cat /tmp/tmp.26Cqly6JCm
++ rm /tmp/tmp.sqMbTZYZwv /tmp/tmp.26Cqly6JCm
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ echo pxc-client-674d6c4f45-9z9z4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ for i in $(seq 0 $((CLUSTER_SIZE - 1)))
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZxMPB7G41j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6JqFgoJSwk
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZxMPB7G41j
++ cat /tmp/tmp.6JqFgoJSwk
++ rm /tmp/tmp.ZxMPB7G41j /tmp/tmp.6JqFgoJSwk
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ echo pxc-client-674d6c4f45-9z9z4
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ kubectl_bin delete -f /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Aa7hfGrxoK
++ mktemp
+ local LAST_ERR=/tmp/tmp.cW0KAdEPzE
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Aa7hfGrxoK
perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.cW0KAdEPzE
+ rm /tmp/tmp.Aa7hfGrxoK /tmp/tmp.cW0KAdEPzE
+ return 0
+ kubectl_bin delete pvc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.XV9wiQC7Tm
++ mktemp
+ local LAST_ERR=/tmp/tmp.bqvwL7o2IU
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pvc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XV9wiQC7Tm
persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-28785 namespace
persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-28785 namespace
persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.bqvwL7o2IU
+ rm /tmp/tmp.XV9wiQC7Tm /tmp/tmp.bqvwL7o2IU
+ return 0
+ desc 'PXC cluster update with explicitly specified image inside version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
PXC cluster update with explicitly specified image inside version service
-----------------------------------------------------------------------------------

+ spinup_pxc smart-update /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local cluster=smart-update
+ local config=/tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.wJGLvoU47I
++ mktemp
+ local LAST_ERR=/tmp/tmp.TEeEY4vpdp
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.wJGLvoU47I
secret/my-cluster-secrets unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.TEeEY4vpdp
+ rm /tmp/tmp.wJGLvoU47I /tmp/tmp.TEeEY4vpdp
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ local pvc_name=
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml
+ local LAST_OUT=/tmp/tmp.6yVF7xnzLi
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-28785~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
++ mktemp
+ local LAST_ERR=/tmp/tmp.MnwIrPlmcw
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6yVF7xnzLi
deployment.apps/pxc-client unchanged
+ cat /tmp/tmp.MnwIrPlmcw
+ rm /tmp/tmp.6yVF7xnzLi /tmp/tmp.MnwIrPlmcw
+ return 0
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ apply_config /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local config_file=/tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local pvc_name=
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml ''
+ local input_file=/tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ local pvc_name=
++ mktemp
+ local LAST_OUT=/tmp/tmp.0JQgmzIkNS
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.4.5-5.1#'
+ cat /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
++ mktemp
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-28785~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.4.5-5.1#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#'
+ local LAST_ERR=/tmp/tmp.12rZ0EydWu
+ local exit_status=0
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.0JQgmzIkNS
perconaxtradbcluster.pxc.percona.com/smart-update created
+ cat /tmp/tmp.12rZ0EydWu
+ rm /tmp/tmp.0JQgmzIkNS /tmp/tmp.12rZ0EydWu
+ return 0
+ desc 'waiting pods to be running for smart-update'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting pods to be running for smart-update
-----------------------------------------------------------------------------------

++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DWui7Ua9Fi
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Fhg5lxW2rF
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.DWui7Ua9Fi
+++ cat /tmp/tmp.Fhg5lxW2rF
+++ rm /tmp/tmp.DWui7Ua9Fi /tmp/tmp.Fhg5lxW2rF
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ local proxy=smart-update-haproxy
++ get_proxy_size smart-update
++ local cluster=smart-update
++ local size=0
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dtbRu2Or6Q
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.M0TFMHGFgk
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.dtbRu2Or6Q
+++ cat /tmp/tmp.M0TFMHGFgk
+++ rm /tmp/tmp.dtbRu2Or6Q /tmp/tmp.M0TFMHGFgk
+++ return 0
++ [[ true == \t\r\u\e ]]
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.size}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.covqiDiFx4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.08p4c26cyS
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.size}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.covqiDiFx4
+++ cat /tmp/tmp.08p4c26cyS
+++ rm /tmp/tmp.covqiDiFx4 /tmp/tmp.08p4c26cyS
+++ return 0
++ size=2
++ echo 2
+ local proxy_size=2
+ wait_for_running smart-update-pxc 3
+ local name=smart-update-pxc
+ local size=3
+ local last_pod
+ local max_retry=480
+ [[ -z smart-update-pxc ]]
+ [[ 3 -le 0 ]]
+ last_pod=2
+ desc 'Waiting for 3 running pods for smart-update-pxc'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 3 running pods for smart-update-pxc
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-pxc-0 480
+ local pod=smart-update-pxc-0
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo smart-update-pxc-0
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-0 condition met
waiting for pod/smart-update-pxc-0 to become Ready.Ok
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-pxc-1 480
+ local pod=smart-update-pxc-1
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ echo smart-update-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-1 condition met
waiting for pod/smart-update-pxc-1 to become Ready.Ok
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-pxc-2 480
+ local pod=smart-update-pxc-2
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo smart-update-pxc-2
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-2 condition met
waiting for pod/smart-update-pxc-2 to become Ready.Ok
+ wait_for_running smart-update-haproxy 2
+ local name=smart-update-haproxy
+ local size=2
+ local last_pod
+ local max_retry=480
+ [[ -z smart-update-haproxy ]]
+ [[ 2 -le 0 ]]
+ last_pod=1
+ desc 'Waiting for 2 running pods for smart-update-haproxy'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 2 running pods for smart-update-haproxy
-----------------------------------------------------------------------------------

++ seq 0 1
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-haproxy-0 480
+ local pod=smart-update-haproxy-0
+ local max_retry=480
+ local ns=
++ echo smart-update-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/smart-update-haproxy-0 condition met
waiting for pod/smart-update-haproxy-0 to become Ready.Ok
+ for i in $(seq 0 $last_pod)
+ wait_pod smart-update-haproxy-1 480
+ local pod=smart-update-haproxy-1
+ local max_retry=480
+ local ns=
++ echo smart-update-haproxy-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/smart-update-haproxy-1 condition met
waiting for pod/smart-update-haproxy-1 to become Ready.Ok
+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local pxc_size=3
+ local proxy_size=2
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local max_seconds=1500
+ local delay=5
+ local retries=300
+ local pxc_ready
+ local proxy_ready
+ sleep 7
+ echo -n 'waiting for pxc/smart-update to be ready'
waiting for pxc/smart-update to be ready++ seq 1 300
+ for _ in $(seq 1 "${retries}")
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sioiRO3Vwn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wmTzBg1kgQ
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sioiRO3Vwn
++ cat /tmp/tmp.wmTzBg1kgQ
++ rm /tmp/tmp.sioiRO3Vwn /tmp/tmp.wmTzBg1kgQ
++ return 0
+ state=ready
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QvLEpMesVo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HoVtj4WgIM
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.QvLEpMesVo
++ cat /tmp/tmp.HoVtj4WgIM
++ rm /tmp/tmp.QvLEpMesVo /tmp/tmp.HoVtj4WgIM
++ return 0
+ pxc_ready=3
+ proxy_ready=0
+ [[ 2 -gt 0 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.bJrc0i31im
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.4GneBDw8mQ
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.bJrc0i31im
+++++ cat /tmp/tmp.4GneBDw8mQ
+++++ rm /tmp/tmp.bJrc0i31im /tmp/tmp.4GneBDw8mQ
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OFGeAJZzPz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k9H4O1Yrt9
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.OFGeAJZzPz
++ cat /tmp/tmp.k9H4O1Yrt9
++ rm /tmp/tmp.OFGeAJZzPz /tmp/tmp.k9H4O1Yrt9
++ return 0
+ proxy_ready=2
+ [[ ready == \r\e\a\d\y ]]
+ [[ 3 == \3 ]]
+ [[ 2 == \2 ]]
+ echo

+ return
+ sleep 10
++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1vcdxgSHMi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bRbZnFO19A
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1vcdxgSHMi
++ cat /tmp/tmp.bRbZnFO19A
++ rm /tmp/tmp.1vcdxgSHMi /tmp/tmp.bRbZnFO19A
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b35HDIn00S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QjPkUm3UA5
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.b35HDIn00S
++ cat /tmp/tmp.QjPkUm3UA5
++ rm /tmp/tmp.b35HDIn00S /tmp/tmp.QjPkUm3UA5
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ echo pxc-client-674d6c4f45-9z9z4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ write_data smart-update 100500 no-proxy -P3306
+ local cluster=smart-update
+ local id=100500
+ local use_proxy=no-proxy
+ local port=-P3306
+ local database=myApp
+ local table=myApp
+ local root_pass
++ get_password smart-update root
++ local cluster=smart-update
++ local user=root
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.OYxAlhttdw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.hxLrqY2LRA
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.OYxAlhttdw
+++ cat /tmp/tmp.hxLrqY2LRA
+++ rm /tmp/tmp.OYxAlhttdw /tmp/tmp.hxLrqY2LRA
+++ return 0
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ base64 --decode
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OJfZhMElzs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RvUWN6AnXo
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.OJfZhMElzs
++ cat /tmp/tmp.RvUWN6AnXo
++ rm /tmp/tmp.OJfZhMElzs /tmp/tmp.RvUWN6AnXo
++ return 0
+ root_pass=root_password
++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.PjWxr9TSvf
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.7E2ifWIbB0
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.PjWxr9TSvf
+++ cat /tmp/tmp.7E2ifWIbB0
+++ rm /tmp/tmp.PjWxr9TSvf /tmp/tmp.7E2ifWIbB0
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ proxy=smart-update-haproxy
+ [[ no-proxy == true ]]
+ [[ no-proxy == \u\s\e\-\p\r\o\x\y ]]
+ host='-h smart-update-pxc-0.smart-update-pxc'
+ run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Uq4RTlUWbF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dcR8yBjYO8
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Uq4RTlUWbF
++ cat /tmp/tmp.dcR8yBjYO8
++ rm /tmp/tmp.Uq4RTlUWbF /tmp/tmp.dcR8yBjYO8
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-674d6c4f45-9z9z4
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
+ desc 'compare initial data'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare initial data
-----------------------------------------------------------------------------------

+ compare_data smart-update select-1 no-proxy -P3306
+ local cluster=smart-update
+ local compare=select-1
+ local use_proxy=no-proxy
+ local port=-P3306
+ local database=myApp
+ local table=myApp
+ local size
+ local root_pass
++ get_password smart-update root
++ local cluster=smart-update
++ local user=root
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WYEfKM2SeP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.7ZdlCud4K7
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.WYEfKM2SeP
+++ cat /tmp/tmp.7ZdlCud4K7
+++ rm /tmp/tmp.WYEfKM2SeP /tmp/tmp.7ZdlCud4K7
+++ return 0
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CHbMpRSvNT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gT2xb0n9Xr
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.CHbMpRSvNT
++ cat /tmp/tmp.gT2xb0n9Xr
++ rm /tmp/tmp.CHbMpRSvNT /tmp/tmp.gT2xb0n9Xr
++ return 0
+ root_pass=root_password
+ [[ no-proxy == true ]]
+ [[ no-proxy == \u\s\e\-\p\r\o\x\y ]]
++ get_pxc_size smart-update
++ local cluster=smart-update
++ local size=0
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.pxc.size}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.xvewXJrjtx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9Q2bKOiIph
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.pxc.size}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.xvewXJrjtx
+++ cat /tmp/tmp.9Q2bKOiIph
+++ rm /tmp/tmp.xvewXJrjtx /tmp/tmp.9Q2bKOiIph
+++ return 0
++ size=3
++ echo 3
+ size=3
++ seq 0 2
+ for i in $(seq 0 $((size - 1)))
+ compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local command_id=select-1
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wtKG1ZFaKp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8Eak8JWfxB
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wtKG1ZFaKp
++ cat /tmp/tmp.8Eak8JWfxB
++ rm /tmp/tmp.wtKG1ZFaKp /tmp/tmp.8Eak8JWfxB
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ echo pxc-client-674d6c4f45-9z9z4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ for i in $(seq 0 $((size - 1)))
+ compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local command_id=select-1
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2TZKFsqrbo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9jge2uS3Cx
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2TZKFsqrbo
++ cat /tmp/tmp.9jge2uS3Cx
++ rm /tmp/tmp.2TZKFsqrbo /tmp/tmp.9jge2uS3Cx
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ echo pxc-client-674d6c4f45-9z9z4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ for i in $(seq 0 $((size - 1)))
+ compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local command_id=select-1
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hP8HR64zow
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FDlfILzCG4
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hP8HR64zow
++ cat /tmp/tmp.FDlfILzCG4
++ rm /tmp/tmp.hP8HR64zow /tmp/tmp.FDlfILzCG4
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ echo pxc-client-674d6c4f45-9z9z4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ is_keyring_plugin_in_use smart-update
+ local cluster=smart-update
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ kubectl exec smart-update-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my
+ return 1
++ jq -r '.versions[].matrix.pxc[].imagePath'
++ grep :8.4
+++ get_operator_pod
+++ local label_prefix=app.kubernetes.io/
++ head -n1
++ sort -V
++ tail -n2
++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++++ grep -c percona-xtradb-cluster-operator
+++ local check_label=1
+++ [[ 1 -eq 0 ]]
+++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
+++ head -1
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ja0c530b7X
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.XKr0i0wo2w
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.ja0c530b7X
+++ cat /tmp/tmp.XKr0i0wo2w
+++ rm /tmp/tmp.ja0c530b7X /tmp/tmp.XKr0i0wo2w
+++ return 0
++ kubectl_bin exec -ti percona-xtradb-cluster-operator-559d7d8944-97rpt -n pxc-operator -- curl -s http://version-service.smart-update2-28785.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XJUoiHJkWH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sJpDnHUPgt
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec -ti percona-xtradb-cluster-operator-559d7d8944-97rpt -n pxc-operator -- curl -s http://version-service.smart-update2-28785.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XJUoiHJkWH
++ cat /tmp/tmp.sJpDnHUPgt
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.XJUoiHJkWH /tmp/tmp.sJpDnHUPgt
++ return 0
+ vs_image=percona/percona-xtradb-cluster:8.4.5-5.1
++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password'
++ local 'command=SELECT @@hostname hostname;'
++ local 'uri=-h smart-update-haproxy -uroot -proot_password'
+++ get_client_pod
+++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.PJaTlvZ4nL
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.4R5Maw3msn
+++ local exit_status=0
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.PJaTlvZ4nL
+++ cat /tmp/tmp.4R5Maw3msn
+++ rm /tmp/tmp.PJaTlvZ4nL /tmp/tmp.4R5Maw3msn
+++ return 0
++ client_pod=pxc-client-674d6c4f45-9z9z4
++ wait_pod pxc-client-674d6c4f45-9z9z4
++ local pod=pxc-client-674d6c4f45-9z9z4
++ local max_retry=480
++ local ns=
+++ echo pxc-client-674d6c4f45-9z9z4
+++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+++ grep -E '^(pxc|proxysql)$'
++ local container=
++ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
++ set +o xtrace
+ initial_primary=smart-update-pxc-0
+ kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.4.5-5.1","schedule": "* * * * *"}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.QOzTYFXtYm
++ mktemp
+ local LAST_ERR=/tmp/tmp.QaBcGcXNCo
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.4.5-5.1","schedule": "* * * * *"}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QOzTYFXtYm
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ cat /tmp/tmp.QaBcGcXNCo
+ rm /tmp/tmp.QOzTYFXtYm /tmp/tmp.QaBcGcXNCo
+ return 0
+ sleep 55
+ check_last_pod_to_update smart-update smart-update-pxc-0 3 percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.4.5-5.1
+ local cluster=smart-update
+ local initial_primary=smart-update-pxc-0
+ local pxc_size=3
+ local target_image=percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.4.5-5.1
+ set +x
Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local pxc_size=3
+ local proxy_size=2
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local max_seconds=1500
+ local delay=5
+ local retries=300
+ local pxc_ready
+ local proxy_ready
+ sleep 7
+ echo -n 'waiting for pxc/smart-update to be ready'
waiting for pxc/smart-update to be ready++ seq 1 300
+ for _ in $(seq 1 "${retries}")
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Q4nTikjNgx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QS6rYJxlDX
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Q4nTikjNgx
++ cat /tmp/tmp.QS6rYJxlDX
++ rm /tmp/tmp.Q4nTikjNgx /tmp/tmp.QS6rYJxlDX
++ return 0
+ state=ready
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zaExoOQ6tH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cbX8awXqIA
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zaExoOQ6tH
++ cat /tmp/tmp.cbX8awXqIA
++ rm /tmp/tmp.zaExoOQ6tH /tmp/tmp.cbX8awXqIA
++ return 0
+ pxc_ready=3
+ proxy_ready=0
+ [[ 2 -gt 0 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.nhexk7MsH8
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.2y3oYOgyhr
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in $(seq 0 2)
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.nhexk7MsH8
+++++ cat /tmp/tmp.2y3oYOgyhr
+++++ rm /tmp/tmp.nhexk7MsH8 /tmp/tmp.2y3oYOgyhr
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qPGg5iUOGE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.C0OsOoBMug
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.qPGg5iUOGE
++ cat /tmp/tmp.C0OsOoBMug
++ rm /tmp/tmp.qPGg5iUOGE /tmp/tmp.C0OsOoBMug
++ return 0
+ proxy_ready=2
+ [[ ready == \r\e\a\d\y ]]
+ [[ 3 == \3 ]]
+ [[ 2 == \2 ]]
+ echo

+ return
++ seq 0 2
+ for i in $(seq 0 $((CLUSTER_SIZE - 1)))
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8K853HjzcF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bm9gmCNyTL
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8K853HjzcF
++ cat /tmp/tmp.bm9gmCNyTL
++ rm /tmp/tmp.8K853HjzcF /tmp/tmp.bm9gmCNyTL
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ echo pxc-client-674d6c4f45-9z9z4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ for i in $(seq 0 $((CLUSTER_SIZE - 1)))
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EXj0yrzQRt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HyMMFr6nlI
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.EXj0yrzQRt
++ cat /tmp/tmp.HyMMFr6nlI
++ rm /tmp/tmp.EXj0yrzQRt /tmp/tmp.HyMMFr6nlI
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ echo pxc-client-674d6c4f45-9z9z4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ for i in $(seq 0 $((CLUSTER_SIZE - 1)))
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1-84.sql ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 8\.0 ]]
+ [[ percona/percona-xtradb-cluster:8.4.5-5.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MQ7mJVh7FS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YfBgBaHMWK
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.MQ7mJVh7FS
++ cat /tmp/tmp.YfBgBaHMWK
++ rm /tmp/tmp.MQ7mJVh7FS /tmp/tmp.YfBgBaHMWK
++ return 0
+ client_pod=pxc-client-674d6c4f45-9z9z4
+ wait_pod pxc-client-674d6c4f45-9z9z4
+ local pod=pxc-client-674d6c4f45-9z9z4
+ local max_retry=480
+ local ns=
++ echo pxc-client-674d6c4f45-9z9z4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-674d6c4f45-9z9z4 condition met
waiting for pod/pxc-client-674d6c4f45-9z9z4 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.kxnkOjofaU/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.kxnkOjofaU/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.kxnkOjofaU/select-1.sql
+ kubectl_bin delete -f /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.g1E21HLyl7
++ mktemp
+ local LAST_ERR=/tmp/tmp.CnceUNfSiy
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /tmp/tmp.kxnkOjofaU/smart-update-version-service-unreachable.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.g1E21HLyl7
perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.CnceUNfSiy
+ rm /tmp/tmp.g1E21HLyl7 /tmp/tmp.CnceUNfSiy
+ return 0
+ kubectl_bin delete pvc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.MEWO9v37JN
++ mktemp
+ local LAST_ERR=/tmp/tmp.IvLERb9xh5
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete pvc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MEWO9v37JN
persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-28785 namespace
persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-28785 namespace
persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.IvLERb9xh5
+ rm /tmp/tmp.MEWO9v37JN /tmp/tmp.IvLERb9xh5
+ return 0
+ desc cleanup
+ set +o xtrace


-----------------------------------------------------------------------------------
cleanup
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/vs.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.HxltsA2gIQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.1rGYK0kiOl
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/smart-update2/conf/vs.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.HxltsA2gIQ
deployment.apps "version-service" deleted from smart-update2-28785 namespace
service "version-service" deleted from smart-update2-28785 namespace
+ cat /tmp/tmp.1rGYK0kiOl
+ rm /tmp/tmp.HxltsA2gIQ /tmp/tmp.1rGYK0kiOl
+ return 0
+ destroy smart-update2-28785
+ local namespace=smart-update2-28785
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+ grep -v 'get backup status: Job.batch'
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ grep -v 'the object has been modified'
+++ grep -c percona-xtradb-cluster-operator
+ grep -v level=info
+ sort -u
+ tee /tmp/tmp.kxnkOjofaU/operator.log
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ head -1
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ovCQn8Zx1a
+++ mktemp
++ local LAST_ERR=/tmp/tmp.19BaUujWVH
++ local exit_status=0
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ovCQn8Zx1a
++ cat /tmp/tmp.19BaUujWVH
++ rm /tmp/tmp.ovCQn8Zx1a /tmp/tmp.19BaUujWVH
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-559d7d8944-97rpt
++ mktemp
+ local LAST_OUT=/tmp/tmp.cvmTTkGQom
++ mktemp
+ local LAST_ERR=/tmp/tmp.wcW7RCOJrd
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-559d7d8944-97rpt
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.cvmTTkGQom
+ cat /tmp/tmp.wcW7RCOJrd
+ rm /tmp/tmp.cvmTTkGQom /tmp/tmp.wcW7RCOJrd
+ return 0
	/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136
	/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:272
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:509
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:215
2026-10-05T14:46:16.807Z	INFO	setup	Feature gates	{"PXCO_FEATURE_GATES": "", "enabled": ""}
2026-10-05T14:46:16.807Z	INFO	setup	Manager starting up	{"gitCommit": "caf25420ee3d0d26f55c22c9a7661405b27337d1", "gitBranch": "PR-2546-caf25420", "buildTime": "2026-10-05T11:36:48Z", "goVersion": "go1.26.8", "os": "linux", "arch": "amd64"}
2026-10-05T14:46:16.807Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.34.9-gke.1655001"}
2026-10-05T14:46:16.811Z	INFO	setup	Registering Components.
2026-10-05T14:46:16.837Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"}
2026-10-05T14:46:16.837Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2026-10-05T14:46:16.837Z	INFO	controller-runtime.metrics	Starting metrics server
2026-10-05T14:46:16.837Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2026-10-05T14:46:16.837Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2026-10-05T14:46:16.837Z	INFO	controller-runtime.webhook	Starting webhook server
2026-10-05T14:46:16.837Z	INFO	setup	Starting the Cmd.
2026-10-05T14:46:16.837Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2026-10-05T14:46:16.838Z	INFO	controller-runtime.certwatcher	Starting certificate poll+watcher	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"}
2026-10-05T14:46:16.938Z	INFO	Attempting to acquire leader lease...	{"lock": "pxc-operator/08db1feb.percona.com"}
2026-10-05T14:46:38.476Z	DEBUG	events	percona-xtradb-cluster-operator-559d7d8944-97rpt_617a158e-1585-492b-97b7-10f923454a22 became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"f9001e8b-4e23-4e18-9eda-559dbdfc3594","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1791211598469215009"}, "reason": "LeaderElection"}
2026-10-05T14:46:38.476Z	INFO	Starting EventSource	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"}
2026-10-05T14:46:38.476Z	INFO	Successfully acquired lease	{"lock": "pxc-operator/08db1feb.percona.com"}
2026-10-05T14:46:38.477Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2026-10-05T14:46:38.477Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2026-10-05T14:46:38.478Z	INFO	Starting EventSource	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"}
2026-10-05T14:46:38.579Z	INFO	Starting Controller	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"}
2026-10-05T14:46:38.579Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1}
2026-10-05T14:46:38.679Z	INFO	Starting Controller	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"}
2026-10-05T14:46:38.679Z	INFO	Starting Controller	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"}
2026-10-05T14:46:38.679Z	INFO	Starting workers	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1}
2026-10-05T14:46:38.679Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1}
2026-10-05T14:46:43.688Z	ERROR	Update status	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"minimal-cluster","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "minimal-cluster", "reconcileID": "240482a3-5ad4-48e7-a584-ccfa19a5d159", "error": "PerconaXtraDBCluster.pxc.percona.com \"minimal-cluster\" not found"}
2026-10-05T14:47:34.754Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "2cb6dce0-d2b6-4307-9ff3-13fbe905d20e", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-10-05T14:47:34.772Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "2cb6dce0-d2b6-4307-9ff3-13fbe905d20e", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-10-05T14:47:35.338Z	ERROR	Reconciler error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "2cb6dce0-d2b6-4307-9ff3-13fbe905d20e", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"}
2026-10-05T14:47:35.566Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "86fa1da0-383c-4311-950a-e6e0fae8ba98", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-10-05T14:47:35.645Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "86fa1da0-383c-4311-950a-e6e0fae8ba98", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-10-05T14:47:35.715Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "86fa1da0-383c-4311-950a-e6e0fae8ba98", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-10-05T14:47:35.778Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "86fa1da0-383c-4311-950a-e6e0fae8ba98", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-10-05T14:47:35.839Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "86fa1da0-383c-4311-950a-e6e0fae8ba98", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-10-05T14:47:35.921Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "86fa1da0-383c-4311-950a-e6e0fae8ba98", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-10-05T14:47:36.912Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "ae6276de-e9b4-4ea9-8b17-26678bc8f34f", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-10-05T14:47:36.932Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "ae6276de-e9b4-4ea9-8b17-26678bc8f34f", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-10-05T14:48:58.403Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "096d0f13-1ce6-4cb4-9015-4cd5687422a8", "user": "operator"}
2026-10-05T14:48:58.439Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "096d0f13-1ce6-4cb4-9015-4cd5687422a8", "user": "monitor"}
2026-10-05T14:48:58.489Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "096d0f13-1ce6-4cb4-9015-4cd5687422a8"}
2026-10-05T14:48:58.529Z	INFO	monitor user privileges granted	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "096d0f13-1ce6-4cb4-9015-4cd5687422a8"}
2026-10-05T14:48:58.556Z	INFO	monitor user audit admin privilege granted	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "096d0f13-1ce6-4cb4-9015-4cd5687422a8"}
2026-10-05T14:48:58.595Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "096d0f13-1ce6-4cb4-9015-4cd5687422a8", "user": "xtrabackup"}
2026-10-05T14:48:58.636Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "096d0f13-1ce6-4cb4-9015-4cd5687422a8"}
2026-10-05T14:48:58.678Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "096d0f13-1ce6-4cb4-9015-4cd5687422a8", "user": "replication"}
2026-10-05T14:51:38.822Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "b86ad04c-e18f-4cfb-9761-5ebc018d9777", "user": "root"}
2026-10-05T14:51:39.899Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "b86ad04c-e18f-4cfb-9761-5ebc018d9777", "new version": "8.4.5-5.1"}
2026-10-05T14:55:34.867Z	INFO	add new job	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "9afe19a0-a0e5-4e71-9640-c3d02c22a485", "name": "ensure-version/smart-update2-28785/smart-update", "schedule": "* * * * *"}
2026-10-05T14:56:34.001Z	DEBUG	Use version service endpoint	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "9afe19a0-a0e5-4e71-9640-c3d02c22a485", "endpoint": "http://version-service.smart-update2-28785.svc.cluster.local:11000"}
2026-10-05T14:56:34.060Z	ERROR	failed to ensure version	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "9afe19a0-a0e5-4e71-9640-c3d02c22a485", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=eccb4690-6282-412d-abc8-3b6f6c72ad41&databaseVersion=8.4.5-5.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=eccb4690-6282-412d-abc8-3b6f6c72ad41&databaseVersion=8.4.5-5.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"}
2026-10-05T14:57:34.001Z	INFO	cluster is not ready	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "9afe19a0-a0e5-4e71-9640-c3d02c22a485"}
2026-10-05T14:58:03.812Z	ERROR	Update status	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "98dbb117-ee32-4ce7-ab8a-ad236192b646", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"}
2026-10-05T14:58:23.223Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "bb9ff793-8068-45de-bcf6-3222a830006d", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-10-05T14:58:23.242Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "bb9ff793-8068-45de-bcf6-3222a830006d", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-10-05T14:58:23.798Z	ERROR	Reconciler error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "bb9ff793-8068-45de-bcf6-3222a830006d", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"}
2026-10-05T14:58:23.903Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "79e6e9a3-3dbb-48bc-93ef-6c258867c985", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-10-05T14:58:23.954Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "79e6e9a3-3dbb-48bc-93ef-6c258867c985", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-10-05T14:58:24.024Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "79e6e9a3-3dbb-48bc-93ef-6c258867c985", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-10-05T14:58:24.119Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "79e6e9a3-3dbb-48bc-93ef-6c258867c985", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-10-05T14:58:24.180Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "79e6e9a3-3dbb-48bc-93ef-6c258867c985", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-10-05T14:58:24.311Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "79e6e9a3-3dbb-48bc-93ef-6c258867c985", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-10-05T14:58:25.161Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "e7087ff0-3174-4f25-95c9-f0f5187c0c34", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-10-05T14:58:25.184Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "e7087ff0-3174-4f25-95c9-f0f5187c0c34", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-10-05T15:02:18.240Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "288ca81e-51f6-41d0-823d-348d8a0eaa5f", "new version": "8.4.5-5.1"}
2026-10-05T15:05:58.199Z	INFO	add new job	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "2674112f-c421-4622-90a8-077655024e70", "name": "ensure-version/smart-update2-28785/smart-update", "schedule": "* * * * *"}
2026-10-05T15:06:58.000Z	DEBUG	Use version service endpoint	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "2674112f-c421-4622-90a8-077655024e70", "endpoint": "http://version-service.smart-update2-28785.svc.cluster.local:11000"}
2026-10-05T15:06:58.033Z	ERROR	failed to ensure version	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "2674112f-c421-4622-90a8-077655024e70", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=4cc50cd1-8cf5-48ff-a321-c457d848b153&databaseVersion=8.4.5-5.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=4cc50cd1-8cf5-48ff-a321-c457d848b153&databaseVersion=8.4.5-5.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"}
2026-10-05T15:07:58.918Z	DEBUG	Use version service endpoint	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "2674112f-c421-4622-90a8-077655024e70", "endpoint": "http://version-service.smart-update2-28785.svc.cluster.local:11000"}
2026-10-05T15:07:58.978Z	ERROR	failed to ensure version	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "2674112f-c421-4622-90a8-077655024e70", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=4cc50cd1-8cf5-48ff-a321-c457d848b153&databaseVersion=8.4.5-5.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=4cc50cd1-8cf5-48ff-a321-c457d848b153&databaseVersion=8.4.5-5.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"}
2026-10-05T15:08:54.413Z	ERROR	Update status	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "de52abb7-6773-4a62-bea7-7d2839330316", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"}
2026-10-05T15:08:58.001Z	INFO	cluster is not found, deleting the job	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "2674112f-c421-4622-90a8-077655024e70", "name": "ensure-version/smart-update2-28785/smart-update", "cluster": "smart-update", "namespace": "smart-update2-28785"}
2026-10-05T15:09:22.397Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "a2393246-2b81-4624-94b9-3e569755c73e", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-10-05T15:09:22.415Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "a2393246-2b81-4624-94b9-3e569755c73e", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-10-05T15:09:22.982Z	ERROR	Reconciler error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "a2393246-2b81-4624-94b9-3e569755c73e", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:580\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:581\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"}
2026-10-05T15:09:23.096Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "6471d557-36e8-42f6-a77e-2ac7c4130e62", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-10-05T15:09:23.148Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "6471d557-36e8-42f6-a77e-2ac7c4130e62", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-10-05T15:09:23.270Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "6471d557-36e8-42f6-a77e-2ac7c4130e62", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-10-05T15:09:23.338Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "6471d557-36e8-42f6-a77e-2ac7c4130e62", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-10-05T15:09:23.376Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "6471d557-36e8-42f6-a77e-2ac7c4130e62", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-10-05T15:09:23.490Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "6471d557-36e8-42f6-a77e-2ac7c4130e62", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-10-05T15:09:24.254Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "bde591b7-73e3-4f8b-9d4e-f8e807d0d966", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-10-05T15:09:24.286Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "bde591b7-73e3-4f8b-9d4e-f8e807d0d966", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-10-05T15:10:43.115Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "3cb2d237-d368-428e-9b0f-9e6a3275cc70", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.231.96:3306: connect: connection refused"}
2026-10-05T15:10:45.264Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "fa61be6c-3b72-4910-8967-a09996f13882", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.231.96:3306: connect: connection refused"}
2026-10-05T15:13:22.518Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "23800b4c-a60f-47af-bc25-9c91bea131f5", "new version": "8.4.5-5.1"}
2026-10-05T15:15:37.869Z	INFO	add new job	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "1211fb03-c475-4fa9-b294-7b2f73ffb577", "name": "ensure-version/smart-update2-28785/smart-update", "schedule": "* * * * *"}
2026-10-05T15:16:37.000Z	DEBUG	Use version service endpoint	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "1211fb03-c475-4fa9-b294-7b2f73ffb577", "endpoint": "http://version-service.smart-update2-28785.svc.cluster.local:11000"}
2026-10-05T15:16:37.045Z	ERROR	failed to ensure version	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "1211fb03-c475-4fa9-b294-7b2f73ffb577", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.4.5-5.1?clusterWideEnabled=true&customResourceUid=0757aea7-c640-42d5-b77a-e9c5dd72911d&databaseVersion=8.4.5-5.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.4.5-5.1?clusterWideEnabled=true&customResourceUid=0757aea7-c640-42d5-b77a-e9c5dd72911d&databaseVersion=8.4.5-5.1&kubeVersion=v1.34.9-gke.1655001&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:284\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:301\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:303\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:213\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"}
2026-10-05T15:17:41.498Z	ERROR	Update status	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "ecc145a3-1bce-4d31-8a61-754e3ec8a20e", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"}
2026-10-05T15:17:41.498Z	INFO	cluster is not found, deleting the job	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-28785"}, "namespace": "smart-update2-28785", "name": "smart-update", "reconcileID": "1211fb03-c475-4fa9-b294-7b2f73ffb577", "name": "ensure-version/smart-update2-28785/smart-update", "cluster": "smart-update", "namespace": "smart-update2-28785"}
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1
github.com/robfig/cron/v3.(*Cron).startJob.func1
github.com/robfig/cron/v3.FuncJob.Run
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
+ local backup_names
++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}'
+ backup_names=
+ [[ -n '' ]]
+ local pxc_names
++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}'
+ pxc_names=
+ [[ -n '' ]]
+ timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found
No resources found
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.EcywhqvFka
++ mktemp
+ local LAST_ERR=/tmp/tmp.fIqmPVX11X
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EcywhqvFka
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.fIqmPVX11X
+ rm /tmp/tmp.EcywhqvFka /tmp/tmp.fIqmPVX11X
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml
namespace "cert-manager" deleted
customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted
serviceaccount "cert-manager-cainjector" deleted from cert-manager namespace
serviceaccount "cert-manager" deleted from cert-manager namespace
serviceaccount "cert-manager-webhook" deleted from cert-manager namespace
clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted
role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace
role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace
role.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace
role.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace
rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace
rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace
rolebinding.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace
rolebinding.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace
service "cert-manager-cainjector" deleted from cert-manager namespace
service "cert-manager" deleted from cert-manager namespace
service "cert-manager-webhook" deleted from cert-manager namespace
deployment.apps "cert-manager-cainjector" deleted from cert-manager namespace
deployment.apps "cert-manager" deleted from cert-manager namespace
deployment.apps "cert-manager-webhook" deleted from cert-manager namespace
mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted
validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace smart-update2-28785
++ mktemp
++ date +%s
+ local LAST_OUT=/tmp/tmp.9ydZtXFJFq
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
+ local deadline=1791213627
++ kubectl_bin get namespace smart-update2-28785 --ignore-not-found -o name
+ local LAST_ERR=/tmp/tmp.jJkq6VOQKU
+ local exit_status=0
++ mktemp
+ local LAST_OUT=/tmp/tmp.OUyklKi5Lx
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace smart-update2-28785
++ mktemp
+ local LAST_ERR=/tmp/tmp.zd7rcq4Awv
+ local exit_status=0
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator
+ [[ -z namespace/smart-update2-28785 ]]
++ date +%s
+ [[ 1791213509 -ge 1791213627 ]]
+ sleep 5
++ kubectl_bin get namespace smart-update2-28785 --ignore-not-found -o name
+ [[ -z namespace/smart-update2-28785 ]]
++ date +%s
+ [[ 1791213515 -ge 1791213627 ]]
+ sleep 5
++ kubectl_bin get namespace smart-update2-28785 --ignore-not-found -o name
+ [[ -z namespace/smart-update2-28785 ]]
++ date +%s
+ [[ 1791213521 -ge 1791213627 ]]
+ sleep 5
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.OUyklKi5Lx
namespace "pxc-operator" force deleted
+ cat /tmp/tmp.zd7rcq4Awv
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
+ rm /tmp/tmp.OUyklKi5Lx /tmp/tmp.zd7rcq4Awv
+ return 0
++ kubectl_bin get namespace smart-update2-28785 --ignore-not-found -o name
+ [[ -z namespace/smart-update2-28785 ]]
++ date +%s
+ [[ 1791213527 -ge 1791213627 ]]
+ sleep 5
++ kubectl_bin get namespace smart-update2-28785 --ignore-not-found -o name
+ [[ -z namespace/smart-update2-28785 ]]
++ date +%s
+ [[ 1791213533 -ge 1791213627 ]]
+ sleep 5
++ kubectl_bin get namespace smart-update2-28785 --ignore-not-found -o name
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.9ydZtXFJFq
namespace "smart-update2-28785" force deleted
+ cat /tmp/tmp.jJkq6VOQKU
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
+ rm /tmp/tmp.9ydZtXFJFq /tmp/tmp.jJkq6VOQKU
+ return 0
+ [[ -z '' ]]
++ date +%s
+ deadline=1791213659
++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}'
++ awk '$2 != ""'
+ [[ -z '' ]]
+ rm -rf /tmp/tmp.kxnkOjofaU
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------