Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/logs/smart-update2-8-0.log
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
+ API=pxc.percona.com/v9-9-9
+ TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ CLUSTER=smart-update
+ CLUSTER_SIZE=3
+ PROXY_SIZE=2
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]]
++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/'
++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ PXC_VER=8.0
+ TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ VS_URL=http://version-service
+ VS_PORT=11000
+ VS_ENDPOINT=http://version-service:11000
+ [[ 8.0 == \8\.\4 ]]
+ VS_UPDATE_STRATEGY=recommended
+ main
+ create_infra smart-update2-11189
+ local ns=smart-update2-11189
+ '[' -n pxc-operator ']'
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get pxc --all-namespaces -o wide
+ kubectl patch pxc -n smart-update2-20345 minimal-cluster --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/minimal-cluster patched (no change)
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.Vnnsy2DRg8
++ mktemp
+ local LAST_ERR=/tmp/tmp.iXhqmxJzzP
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Vnnsy2DRg8
perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-20345 namespace
+ cat /tmp/tmp.iXhqmxJzzP
+ rm /tmp/tmp.Vnnsy2DRg8 /tmp/tmp.iXhqmxJzzP
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.5uQkgBNOTM
++ mktemp
+ local LAST_ERR=/tmp/tmp.83RlqbcRpZ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5uQkgBNOTM
No resources found
+ cat /tmp/tmp.83RlqbcRpZ
+ rm /tmp/tmp.5uQkgBNOTM /tmp/tmp.83RlqbcRpZ
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.J0jKxDK6np
++ mktemp
+ local LAST_ERR=/tmp/tmp.ECoP0c3yZu
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.J0jKxDK6np
No resources found
+ cat /tmp/tmp.ECoP0c3yZu
+ rm /tmp/tmp.J0jKxDK6np /tmp/tmp.ECoP0c3yZu
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get MutatingWebhookConfiguration
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl api-resources
++ grep chaos-mesh.org
++ kubectl get crd
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
++ mktemp
+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.NBY6UiEmvR
+ local LAST_OUT=/tmp/tmp.ne3tsDgIhc
++ mktemp
+ awk '{print$1}'
++ mktemp
+ xargs kubectl delete ns
+ local LAST_ERR=/tmp/tmp.Sa9F7cfUgU
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.vNmTyuhEQp
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ne3tsDgIhc
+ cat /tmp/tmp.Sa9F7cfUgU
+ rm /tmp/tmp.ne3tsDgIhc /tmp/tmp.Sa9F7cfUgU
+ return 0
namespace "cert-manager" deleted
namespace "smart-update2-20345" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.NBY6UiEmvR
namespace "pxc-operator" deleted
+ cat /tmp/tmp.vNmTyuhEQp
+ rm /tmp/tmp.NBY6UiEmvR /tmp/tmp.vNmTyuhEQp
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.2S9RK9nMxZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.pwEH4qIo6A
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2S9RK9nMxZ
namespace/pxc-operator created
+ cat /tmp/tmp.pwEH4qIo6A
+ rm /tmp/tmp.2S9RK9nMxZ /tmp/tmp.pwEH4qIo6A
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Qf5X0g0xZB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ybCskmbKN6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Qf5X0g0xZB
++ cat /tmp/tmp.ybCskmbKN6
++ rm /tmp/tmp.Qf5X0g0xZB /tmp/tmp.ybCskmbKN6
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster7 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.TRv08IEki0
++ mktemp
+ local LAST_ERR=/tmp/tmp.4DMs1uXpLZ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster7 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.TRv08IEki0
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster7" modified.
+ cat /tmp/tmp.4DMs1uXpLZ
+ rm /tmp/tmp.TRv08IEki0 /tmp/tmp.4DMs1uXpLZ
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.oh2VR6petq
++ mktemp
+ local LAST_ERR=/tmp/tmp.3XdqCAKw53
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.oh2VR6petq
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.3XdqCAKw53
+ rm /tmp/tmp.oh2VR6petq /tmp/tmp.3XdqCAKw53
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/cw-rbac.yaml
+ kubectl_bin apply -f -
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
++ mktemp
+ local LAST_OUT=/tmp/tmp.oCLo5g4HKK
++ mktemp
+ local LAST_ERR=/tmp/tmp.eEtQVtdcmA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.oCLo5g4HKK
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.eEtQVtdcmA
+ rm /tmp/tmp.oCLo5g4HKK /tmp/tmp.eEtQVtdcmA
+ return 0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/cw-operator.yaml
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' -
+ kubectl_bin apply -f -
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
++ mktemp
+ local LAST_OUT=/tmp/tmp.JhgTe60CsN
++ mktemp
+ local LAST_ERR=/tmp/tmp.GtZ33zKuyX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.JhgTe60CsN
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.GtZ33zKuyX
+ rm /tmp/tmp.JhgTe60CsN /tmp/tmp.GtZ33zKuyX
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.TrnAwzmfsB
++ mktemp
+ local LAST_ERR=/tmp/tmp.DTEbWmtlAW
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.TrnAwzmfsB
pod/percona-xtradb-cluster-operator-665ff7485-dr47d condition met
+ cat /tmp/tmp.DTEbWmtlAW
+ rm /tmp/tmp.TrnAwzmfsB /tmp/tmp.DTEbWmtlAW
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ head -1
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kZjFmfVlP6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dFxKLf6JCU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.kZjFmfVlP6
++ cat /tmp/tmp.dFxKLf6JCU
++ rm /tmp/tmp.kZjFmfVlP6 /tmp/tmp.dFxKLf6JCU
++ return 0
+ wait_pod percona-xtradb-cluster-operator-665ff7485-dr47d 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-665ff7485-dr47d
+ local max_retry=480
+ local ns=pxc-operator
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo percona-xtradb-cluster-operator-665ff7485-dr47d
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-665ff7485-dr47d condition met
waiting for pod/percona-xtradb-cluster-operator-665ff7485-dr47d to become Ready.Ok
+ sleep 3
+ create_namespace smart-update2-11189
+ local namespace=smart-update2-11189
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get MutatingWebhookConfiguration
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl api-resources
++ grep chaos-mesh.org
++ kubectl get crd
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrole
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces smart-update2-11189'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces smart-update2-11189
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace smart-update2-11189
+ xargs kubectl delete ns
+ awk '{print$1}'
++ mktemp
+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ local LAST_OUT=/tmp/tmp.nyEft69Uyx
++ mktemp
+ local LAST_OUT=/tmp/tmp.lB18KfnDdF
++ mktemp
+ local LAST_ERR=/tmp/tmp.V9XQ1VyTjB
+ local exit_status=0
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.i9hABiv4MT
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace smart-update2-11189
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lB18KfnDdF
+ cat /tmp/tmp.V9XQ1VyTjB
+ rm /tmp/tmp.lB18KfnDdF /tmp/tmp.V9XQ1VyTjB
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace smart-update2-11189
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace smart-update2-11189
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.nyEft69Uyx
+ cat /tmp/tmp.i9hABiv4MT
Error from server (NotFound): namespaces "smart-update2-11189" not found
+ rm /tmp/tmp.nyEft69Uyx /tmp/tmp.i9hABiv4MT
+ return 1
+ :
+ wait_for_delete namespace/smart-update2-11189
+ local res=namespace/smart-update2-11189
+ echo -n 'waiting for namespace/smart-update2-11189 to be deleted'
waiting for namespace/smart-update2-11189 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "smart-update2-11189" not found
+ desc 'create namespace smart-update2-11189'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace smart-update2-11189
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace smart-update2-11189
++ mktemp
+ local LAST_OUT=/tmp/tmp.zX4fxjxVAw
++ mktemp
+ local LAST_ERR=/tmp/tmp.HHMzCS0kzT
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace smart-update2-11189
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.zX4fxjxVAw
namespace/smart-update2-11189 created
+ cat /tmp/tmp.HHMzCS0kzT
+ rm /tmp/tmp.zX4fxjxVAw /tmp/tmp.HHMzCS0kzT
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sYhEYqWtau
+++ mktemp
++ local LAST_ERR=/tmp/tmp.t0iaqt5hnO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sYhEYqWtau
++ cat /tmp/tmp.t0iaqt5hnO
++ rm /tmp/tmp.sYhEYqWtau /tmp/tmp.t0iaqt5hnO
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster7 --namespace=smart-update2-11189
++ mktemp
+ local LAST_OUT=/tmp/tmp.QYQH8Kp3rl
++ mktemp
+ local LAST_ERR=/tmp/tmp.60t9bMB18F
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster7 --namespace=smart-update2-11189
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QYQH8Kp3rl
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2434-3b65d7fe-1-cluster7" modified.
+ cat /tmp/tmp.60t9bMB18F
+ rm /tmp/tmp.QYQH8Kp3rl /tmp/tmp.60t9bMB18F
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ff6rTgrBXh
++ mktemp
+ local LAST_ERR=/tmp/tmp.Trs4G6Gvft
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Ff6rTgrBXh
secret/minio-secret created
secret/aws-s3-secret created
secret/do-spaces-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.Trs4G6Gvft
+ rm /tmp/tmp.Ff6rTgrBXh /tmp/tmp.Trs4G6Gvft
+ return 0
+ deploy_version_service
+ desc 'install version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install version service
-----------------------------------------------------------------------------------

+ kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json
++ mktemp
+ local LAST_OUT=/tmp/tmp.bduScuGhJn
++ mktemp
+ local LAST_ERR=/tmp/tmp.QOOAttRJ93
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.bduScuGhJn
configmap/versions created
+ cat /tmp/tmp.QOOAttRJ93
+ rm /tmp/tmp.bduScuGhJn /tmp/tmp.QOOAttRJ93
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/vs.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.8wqeu8oAQL
++ mktemp
+ local LAST_ERR=/tmp/tmp.F5ySwpkg0L
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/vs.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.8wqeu8oAQL
deployment.apps/version-service created
service/version-service created
+ cat /tmp/tmp.F5ySwpkg0L
+ rm /tmp/tmp.8wqeu8oAQL /tmp/tmp.F5ySwpkg0L
+ return 0
+ sleep 10
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.IkIDcJ14yp
++ mktemp
+ local LAST_ERR=/tmp/tmp.5arso2S8xP
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.IkIDcJ14yp
namespace/cert-manager created
+ cat /tmp/tmp.5arso2S8xP
+ rm /tmp/tmp.IkIDcJ14yp /tmp/tmp.5arso2S8xP
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.Dr1vjeHQv7
++ mktemp
+ local LAST_ERR=/tmp/tmp.g2mgECTiHe
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Dr1vjeHQv7
namespace/cert-manager labeled
+ cat /tmp/tmp.g2mgECTiHe
+ rm /tmp/tmp.Dr1vjeHQv7 /tmp/tmp.g2mgECTiHe
+ return 0
+ kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.qGLKUFuYT2
++ mktemp
+ local LAST_ERR=/tmp/tmp.CkNslkvHPp
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.qGLKUFuYT2
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-tokenrequest created
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager-cainjector created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.CkNslkvHPp
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.qGLKUFuYT2 /tmp/tmp.CkNslkvHPp
+ return 0
+ '[' '' == 4.10 ']'
+ sleep 70
++ sort -V
++ grep :8.0
++ jq -r '.versions[].matrix.pxc[].imagePath'
++ tail -n3
++ head -n1
+++ get_operator_pod
+++ local label_prefix=app.kubernetes.io/
++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++++ grep -c percona-xtradb-cluster-operator
+++ local check_label=1
+++ [[ 1 -eq 0 ]]
+++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
+++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++++ mktemp
+++ head -1
+++ local LAST_OUT=/tmp/tmp.0YVnRagzoS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bipMH3pVM9
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.0YVnRagzoS
+++ cat /tmp/tmp.bipMH3pVM9
+++ rm /tmp/tmp.0YVnRagzoS /tmp/tmp.bipMH3pVM9
+++ return 0
++ kubectl_bin exec -ti percona-xtradb-cluster-operator-665ff7485-dr47d -n pxc-operator -- curl -s http://version-service.smart-update2-11189.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FezP9vJHIZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6EuHgwVMUv
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -ti percona-xtradb-cluster-operator-665ff7485-dr47d -n pxc-operator -- curl -s http://version-service.smart-update2-11189.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FezP9vJHIZ
++ cat /tmp/tmp.6EuHgwVMUv
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.FezP9vJHIZ /tmp/tmp.6EuHgwVMUv
++ return 0
+ IMAGE_PXC=percona/percona-xtradb-cluster:8.0.41-32.1
+ kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.MZiLOJ0ZQi
++ mktemp
+ local LAST_ERR=/tmp/tmp.67SCpvznua
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MZiLOJ0ZQi
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched
+ cat /tmp/tmp.67SCpvznua
+ rm /tmp/tmp.MZiLOJ0ZQi /tmp/tmp.67SCpvznua
+ return 0
+ kubectl_bin -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-11189.svc.cluster.local:11000
++ mktemp
+ local LAST_OUT=/tmp/tmp.z5kID1fbyI
++ mktemp
+ local LAST_ERR=/tmp/tmp.SfGFNk4BGh
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-11189.svc.cluster.local:11000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.z5kID1fbyI
deployment.apps/percona-xtradb-cluster-operator env updated
+ cat /tmp/tmp.SfGFNk4BGh
+ rm /tmp/tmp.z5kID1fbyI /tmp/tmp.SfGFNk4BGh
+ return 0
+ desc 'Starting telemetry testing'
+ set +o xtrace


-----------------------------------------------------------------------------------
Starting telemetry testing
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f -
+ /usr/bin/sed s/version-service/version-service-cr/g /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/vs.yml
++ yq 'select(.kind == "Deployment").spec.template.spec.containers[0].image' /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/vs.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.FWnUmUGhue
++ mktemp
+ local LAST_ERR=/tmp/tmp.i3QH9akhzg
+ local exit_status=0
+ yq eval '(. | select(.kind == "Deployment") | .spec.template.spec.containers[0].image) = "perconalab/version-service:main-latest"'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.FWnUmUGhue
deployment.apps/version-service-cr created
service/version-service-cr created
+ cat /tmp/tmp.i3QH9akhzg
+ rm /tmp/tmp.FWnUmUGhue /tmp/tmp.i3QH9akhzg
+ return 0
+ kubectl_bin delete pod -l run=version-service
++ mktemp
+ local LAST_OUT=/tmp/tmp.Em5mfhkDFr
++ mktemp
+ local LAST_ERR=/tmp/tmp.JGQTBjBL7X
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -l run=version-service
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Em5mfhkDFr
pod "version-service-559c4df55c-gdvjl" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.JGQTBjBL7X
+ rm /tmp/tmp.Em5mfhkDFr /tmp/tmp.JGQTBjBL7X
+ return 0
++ sed -r 's/^.*:([0-9]+.[0-9]+).*/\1/'
++ echo -n percona/percona-xtradb-cluster:8.0.41-32.1
+ IMAGE_PREFIX=8.0
+ desc 'Enable telemetry on operator level'
+ set +o xtrace


-----------------------------------------------------------------------------------
Enable telemetry on operator level
-----------------------------------------------------------------------------------

+ kubectl_bin apply -n pxc-operator -f -
+ yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "false"'
+ kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.fddLJNojNZ
++ mktemp
+ local LAST_OUT=/tmp/tmp.iR2ocZNyq6
++ mktemp
+ local LAST_ERR=/tmp/tmp.WAwSNKzxKT
+ local exit_status=0
++ mktemp
+ local LAST_ERR=/tmp/tmp.y4CJIRU0ya
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n pxc-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.iR2ocZNyq6
+ cat /tmp/tmp.y4CJIRU0ya
+ rm /tmp/tmp.iR2ocZNyq6 /tmp/tmp.y4CJIRU0ya
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.fddLJNojNZ
deployment.apps/percona-xtradb-cluster-operator configured
+ cat /tmp/tmp.WAwSNKzxKT
+ rm /tmp/tmp.fddLJNojNZ /tmp/tmp.WAwSNKzxKT
+ return 0
+ sleep 30
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ head -1
+++ mktemp
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ local LAST_OUT=/tmp/tmp.T2PFriO3z6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QPUcwif6Rw
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.T2PFriO3z6
++ cat /tmp/tmp.QPUcwif6Rw
++ rm /tmp/tmp.T2PFriO3z6 /tmp/tmp.QPUcwif6Rw
++ return 0
+ wait_pod percona-xtradb-cluster-operator-7dcd5544-jklfw 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-7dcd5544-jklfw
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-7dcd5544-jklfw
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-7dcd5544-jklfw condition met
waiting for pod/percona-xtradb-cluster-operator-7dcd5544-jklfw to become Ready.Ok
+ check_telemetry_transfer http://version-service-cr.smart-update2-11189.svc.cluster.local:11000 disabled enabled
+ local cr_vs_uri=http://version-service-cr.smart-update2-11189.svc.cluster.local:11000
+ local cr_vs_channel=disabled
+ local telemetry_state=enabled
+ desc 'create PXC minimal cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC minimal cluster
-----------------------------------------------------------------------------------

+ cluster=minimal-cluster
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.HLh8ZpVZEI
++ mktemp
+ local LAST_ERR=/tmp/tmp.d9iCdRDEeW
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.HLh8ZpVZEI
deployment.apps/pxc-client created
+ cat /tmp/tmp.d9iCdRDEeW
+ rm /tmp/tmp.HLh8ZpVZEI /tmp/tmp.d9iCdRDEeW
+ return 0
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml
+ yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.YNfIAWzISZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.gLIHLLsH0N
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.YNfIAWzISZ
secret/minimal-cluster created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.gLIHLLsH0N
+ rm /tmp/tmp.YNfIAWzISZ /tmp/tmp.gLIHLLsH0N
+ return 0
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/cr-minimal.yaml
+ yq eval '.spec.upgradeOptions.apply="disabled"'
+ yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"'
+ yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.41-32.1"'
+ yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"'
+ yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe"'
+ kubectl_bin apply -f -
+ yq eval '.spec.crVersion="9.9.9"'
+ yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-11189.svc.cluster.local:11000"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.7jYoY8sOWQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.mcuPYs8DTN
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7jYoY8sOWQ
perconaxtradbcluster.pxc.percona.com/minimal-cluster created
+ cat /tmp/tmp.mcuPYs8DTN
+ rm /tmp/tmp.7jYoY8sOWQ /tmp/tmp.mcuPYs8DTN
+ return 0
+ desc 'check if Pod is started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod is started
-----------------------------------------------------------------------------------

+ wait_for_running minimal-cluster-pxc 1
+ local name=minimal-cluster-pxc
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod minimal-cluster-pxc-0 480
+ local pod=minimal-cluster-pxc-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/minimal-cluster-pxc-0 condition met
waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok
+ sleep 20
+ local proxy
++ get_proxy minimal-cluster
++ local target_cluster=minimal-cluster
+++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.USxUjpaG0C
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6NVakNftR1
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.USxUjpaG0C
+++ cat /tmp/tmp.6NVakNftR1
+++ rm /tmp/tmp.USxUjpaG0C /tmp/tmp.6NVakNftR1
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo minimal-cluster-haproxy
++ return
+ proxy=minimal-cluster-haproxy
+ wait_for_running minimal-cluster-haproxy 1
+ local name=minimal-cluster-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod minimal-cluster-haproxy-0 480
+ local pod=minimal-cluster-haproxy-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/minimal-cluster-haproxy-0 condition met
waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.I9VmnxC2fB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.befhHwWAVZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.I9VmnxC2fB
++ cat /tmp/tmp.befhHwWAVZ
++ rm /tmp/tmp.I9VmnxC2fB /tmp/tmp.befhHwWAVZ
++ return 0
+ client_pod=pxc-client-67fc4995bb-pcz8q
+ wait_pod pxc-client-67fc4995bb-pcz8q
+ local pod=pxc-client-67fc4995bb-pcz8q
+ local max_retry=480
+ local ns=
++ echo pxc-client-67fc4995bb-pcz8q
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-pcz8q condition met
waiting for pod/pxc-client-67fc4995bb-pcz8q to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument.
command terminated with exit code 5
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u29QxH6dbL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KcHzuG9slQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.u29QxH6dbL
++ cat /tmp/tmp.KcHzuG9slQ
++ rm /tmp/tmp.u29QxH6dbL /tmp/tmp.KcHzuG9slQ
++ return 0
+ client_pod=pxc-client-67fc4995bb-pcz8q
+ wait_pod pxc-client-67fc4995bb-pcz8q
+ local pod=pxc-client-67fc4995bb-pcz8q
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-67fc4995bb-pcz8q
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-pcz8q condition met
waiting for pod/pxc-client-67fc4995bb-pcz8q to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument.
command terminated with exit code 5
+ grep -Eo '\{.*\}'
++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.time_ms")'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ grep -E 'server request payload|unary call'
+ kubectl_bin logs version-service-cr-76d4659869-6jqrz
++ mktemp
+ local LAST_OUT=/tmp/tmp.2vMjsIE0YN
++ mktemp
+ local LAST_ERR=/tmp/tmp.wqd8RK8AvX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs version-service-cr-76d4659869-6jqrz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2vMjsIE0YN
+ cat /tmp/tmp.wqd8RK8AvX
+ rm /tmp/tmp.2vMjsIE0YN /tmp/tmp.wqd8RK8AvX
+ return 0
+ grep -Eo '\{.*\}'
++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}'
+ jq 'del(."grpc.time_ms")'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ grep -E 'server request payload|unary call'
+ kubectl_bin logs version-service-559c4df55c-6gdzv
++ mktemp
+ local LAST_OUT=/tmp/tmp.37U5K7TNSm
++ mktemp
+ local LAST_ERR=/tmp/tmp.XqBYOljdc5
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs version-service-559c4df55c-6gdzv
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.37U5K7TNSm
+ cat /tmp/tmp.XqBYOljdc5
+ rm /tmp/tmp.37U5K7TNSm /tmp/tmp.XqBYOljdc5
+ return 0
+ local telemetry_log_file=enabled_telemetry.version-service-cw.log.json
+ desc 'telemetry was disabled in CR but in operator not'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR but in operator not
-----------------------------------------------------------------------------------

+ '[' disabled == disabled -a enabled == enabled ']'
+ desc 'operator fallback VS should have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
operator fallback VS should have telemetry
-----------------------------------------------------------------------------------

+ diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json /dev/fd/63
++ grep -f /tmp/tmp.oGaB4XvPf0/enabled_telemetry.version-service.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json
+ desc 'CR VS should not have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
CR VS should not have telemetry
-----------------------------------------------------------------------------------

+ [[ -s /tmp/tmp.oGaB4XvPf0/enabled_telemetry.version-service-cr.log.json ]]
+ local image_prefix=disabled
+ image_prefix=disabled
+ local telemetry_cr_log_file=enabled_telemetry.version-service-cr-disabled-cw.log.json
+ desc 'telemetry was disabled in operator but not in CR'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in operator but not in CR
-----------------------------------------------------------------------------------

+ [[ disabled == \d\i\s\a\b\l\e\d\-\r\e\c\o\m\m\e\n\d\e\d ]]
+ [[ disabled == \d\i\s\a\b\l\e\d\-\l\a\t\e\s\t ]]
+ desc 'telemetry was disabled in CR as well as in operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR as well as in operator
-----------------------------------------------------------------------------------

+ '[' disabled == disabled -a enabled == disabled ']'
+ kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.2oTjzI4gYA
++ mktemp
+ local LAST_ERR=/tmp/tmp.a7GGAF019o
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2oTjzI4gYA
perconaxtradbcluster.pxc.percona.com/minimal-cluster patched
+ cat /tmp/tmp.a7GGAF019o
+ rm /tmp/tmp.2oTjzI4gYA /tmp/tmp.a7GGAF019o
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ head -1
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.abC1WY1s2i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YiVTU5tJzN
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.abC1WY1s2i
++ cat /tmp/tmp.YiVTU5tJzN
++ rm /tmp/tmp.abC1WY1s2i /tmp/tmp.YiVTU5tJzN
++ return 0
+ kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-7dcd5544-jklfw
++ mktemp
+ local LAST_OUT=/tmp/tmp.e95fliua7y
++ mktemp
+ local LAST_ERR=/tmp/tmp.0XECSdLGeA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-7dcd5544-jklfw
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.e95fliua7y
pod "percona-xtradb-cluster-operator-7dcd5544-jklfw" deleted from pxc-operator namespace
+ cat /tmp/tmp.0XECSdLGeA
+ rm /tmp/tmp.e95fliua7y /tmp/tmp.0XECSdLGeA
+ return 0
+ kubectl_bin delete pxc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.ASO5EKvBqS
++ mktemp
+ local LAST_ERR=/tmp/tmp.rFH9a7q5GE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ASO5EKvBqS
perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.rFH9a7q5GE
+ rm /tmp/tmp.ASO5EKvBqS /tmp/tmp.rFH9a7q5GE
+ return 0
+ kubectl_bin delete deploy pxc-client
++ mktemp
+ local LAST_OUT=/tmp/tmp.2sbGP4gKXz
++ mktemp
+ local LAST_ERR=/tmp/tmp.MkspGq0i5J
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete deploy pxc-client
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2sbGP4gKXz
deployment.apps "pxc-client" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.MkspGq0i5J
+ rm /tmp/tmp.2sbGP4gKXz /tmp/tmp.MkspGq0i5J
+ return 0
+ sleep 30
+ desc 'Disabling telemetry on the operator level'
+ set +o xtrace


-----------------------------------------------------------------------------------
Disabling telemetry on the operator level
-----------------------------------------------------------------------------------

+ kubectl_bin delete pod -l run=version-service-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.aHrsLILlny
++ mktemp
+ local LAST_ERR=/tmp/tmp.YxcSQ1cxxi
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -l run=version-service-cr
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.aHrsLILlny
pod "version-service-cr-76d4659869-6jqrz" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.YxcSQ1cxxi
+ rm /tmp/tmp.aHrsLILlny /tmp/tmp.YxcSQ1cxxi
+ return 0
+ kubectl_bin delete pod -l run=version-service
++ mktemp
+ local LAST_OUT=/tmp/tmp.oEbXq4o84Y
++ mktemp
+ local LAST_ERR=/tmp/tmp.4wr4MwnC64
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -l run=version-service
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.oEbXq4o84Y
pod "version-service-559c4df55c-6gdzv" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.4wr4MwnC64
+ rm /tmp/tmp.oEbXq4o84Y /tmp/tmp.4wr4MwnC64
+ return 0
+ kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator
+ yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"'
+ kubectl_bin apply -n pxc-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.J5kQq8Dao8
++ mktemp
+ local LAST_OUT=/tmp/tmp.T6y8tZLPDf
++ mktemp
+ local LAST_ERR=/tmp/tmp.bSSBObfZTv
+ local exit_status=0
++ mktemp
+ local LAST_ERR=/tmp/tmp.JdBg6cMVtc
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n pxc-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.J5kQq8Dao8
+ cat /tmp/tmp.bSSBObfZTv
+ rm /tmp/tmp.J5kQq8Dao8 /tmp/tmp.bSSBObfZTv
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.T6y8tZLPDf
deployment.apps/percona-xtradb-cluster-operator configured
+ cat /tmp/tmp.JdBg6cMVtc
+ rm /tmp/tmp.T6y8tZLPDf /tmp/tmp.JdBg6cMVtc
+ return 0
+ sleep 30
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ head -1
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NgzL9sgvJE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NVVkQTXs8U
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NgzL9sgvJE
++ cat /tmp/tmp.NVVkQTXs8U
++ rm /tmp/tmp.NgzL9sgvJE /tmp/tmp.NVVkQTXs8U
++ return 0
+ wait_pod percona-xtradb-cluster-operator-f5ddd9c4f-hfv55 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-f5ddd9c4f-hfv55
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-f5ddd9c4f-hfv55
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-f5ddd9c4f-hfv55 condition met
waiting for pod/percona-xtradb-cluster-operator-f5ddd9c4f-hfv55 to become Ready.Ok
+ check_telemetry_transfer http://version-service-cr.smart-update2-11189.svc.cluster.local:11000 8.0-recommended disabled
+ local cr_vs_uri=http://version-service-cr.smart-update2-11189.svc.cluster.local:11000
+ local cr_vs_channel=8.0-recommended
+ local telemetry_state=disabled
+ desc 'create PXC minimal cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC minimal cluster
-----------------------------------------------------------------------------------

+ cluster=minimal-cluster
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Al7xR6Jm6u
++ mktemp
+ local LAST_ERR=/tmp/tmp.4G1cUDd5CH
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Al7xR6Jm6u
deployment.apps/pxc-client created
+ cat /tmp/tmp.4G1cUDd5CH
+ rm /tmp/tmp.Al7xR6Jm6u /tmp/tmp.4G1cUDd5CH
+ return 0
+ yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"'
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.UZFq5Kd2wU
++ mktemp
+ local LAST_ERR=/tmp/tmp.abZz8CVRbW
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UZFq5Kd2wU
secret/minimal-cluster unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.abZz8CVRbW
+ rm /tmp/tmp.UZFq5Kd2wU /tmp/tmp.abZz8CVRbW
+ return 0
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/cr-minimal.yaml
+ yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-11189.svc.cluster.local:11000"'
+ kubectl_bin apply -f -
+ yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.41-32.1"'
+ yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"'
+ yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"'
+ yq eval '.spec.crVersion="9.9.9"'
+ yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.j8y8NsYKDt
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="8.0-recommended"'
+ local LAST_ERR=/tmp/tmp.ebebay6ntt
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.j8y8NsYKDt
perconaxtradbcluster.pxc.percona.com/minimal-cluster created
+ cat /tmp/tmp.ebebay6ntt
+ rm /tmp/tmp.j8y8NsYKDt /tmp/tmp.ebebay6ntt
+ return 0
+ desc 'check if Pod is started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod is started
-----------------------------------------------------------------------------------

+ wait_for_running minimal-cluster-pxc 1
+ local name=minimal-cluster-pxc
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod minimal-cluster-pxc-0 480
+ local pod=minimal-cluster-pxc-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/minimal-cluster-pxc-0 condition met
waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok
+ sleep 20
+ local proxy
++ get_proxy minimal-cluster
++ local target_cluster=minimal-cluster
+++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.x1iQ1uUFvb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rZJVgpfKkF
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.x1iQ1uUFvb
+++ cat /tmp/tmp.rZJVgpfKkF
+++ rm /tmp/tmp.x1iQ1uUFvb /tmp/tmp.rZJVgpfKkF
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo minimal-cluster-haproxy
++ return
+ proxy=minimal-cluster-haproxy
+ wait_for_running minimal-cluster-haproxy 1
+ local name=minimal-cluster-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod minimal-cluster-haproxy-0 480
+ local pod=minimal-cluster-haproxy-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/minimal-cluster-haproxy-0 condition met
waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G6zrR1oho1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zI2q1gnSXY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.G6zrR1oho1
++ cat /tmp/tmp.zI2q1gnSXY
++ rm /tmp/tmp.G6zrR1oho1 /tmp/tmp.zI2q1gnSXY
++ return 0
+ client_pod=pxc-client-67fc4995bb-2lpzd
+ wait_pod pxc-client-67fc4995bb-2lpzd
+ local pod=pxc-client-67fc4995bb-2lpzd
+ local max_retry=480
+ local ns=
++ echo pxc-client-67fc4995bb-2lpzd
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-2lpzd condition met
waiting for pod/pxc-client-67fc4995bb-2lpzd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument.
command terminated with exit code 5
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FI3qdljQtp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RAoc5CJi2M
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FI3qdljQtp
++ cat /tmp/tmp.RAoc5CJi2M
++ rm /tmp/tmp.FI3qdljQtp /tmp/tmp.RAoc5CJi2M
++ return 0
+ client_pod=pxc-client-67fc4995bb-2lpzd
+ wait_pod pxc-client-67fc4995bb-2lpzd
+ local pod=pxc-client-67fc4995bb-2lpzd
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-67fc4995bb-2lpzd
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-2lpzd condition met
waiting for pod/pxc-client-67fc4995bb-2lpzd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument.
command terminated with exit code 5
+ jq 'del(."grpc.time_ms")'
+ jq 'del(."grpc.start_time")'
+ grep -Eo '\{.*\}'
+ grep -E 'server request payload|unary call'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}'
+ kubectl_bin logs version-service-cr-76d4659869-gssqx
++ mktemp
+ local LAST_OUT=/tmp/tmp.4V5UicT1T2
++ mktemp
+ local LAST_ERR=/tmp/tmp.cgkdAZrLzp
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs version-service-cr-76d4659869-gssqx
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.4V5UicT1T2
+ cat /tmp/tmp.cgkdAZrLzp
+ rm /tmp/tmp.4V5UicT1T2 /tmp/tmp.cgkdAZrLzp
+ return 0
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ jq 'del(."grpc.time_ms")'
+ grep -Eo '\{.*\}'
+ grep -E 'server request payload|unary call'
++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}'
+ kubectl_bin logs version-service-559c4df55c-q4hxf
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ed1Lk3y0Ag
++ mktemp
+ local LAST_ERR=/tmp/tmp.IPsZvx1NGH
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs version-service-559c4df55c-q4hxf
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Ed1Lk3y0Ag
+ cat /tmp/tmp.IPsZvx1NGH
+ rm /tmp/tmp.Ed1Lk3y0Ag /tmp/tmp.IPsZvx1NGH
+ return 0
+ local telemetry_log_file=disabled_telemetry.version-service-cw.log.json
+ desc 'telemetry was disabled in CR but in operator not'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR but in operator not
-----------------------------------------------------------------------------------

+ '[' 8.0-recommended == disabled -a disabled == enabled ']'
+ local image_prefix=8.0
+ image_prefix=8.0
+ local telemetry_cr_log_file=disabled_telemetry.version-service-cr-8.0-cw.log.json
+ desc 'telemetry was disabled in operator but not in CR'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in operator but not in CR
-----------------------------------------------------------------------------------

+ [[ 8.0-recommended == \8\.\0\-\r\e\c\o\m\m\e\n\d\e\d ]]
+ '[' disabled == disabled ']'
+ desc 'cr VS should have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
cr VS should have telemetry
-----------------------------------------------------------------------------------

+ diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.0-cw.log.json /dev/fd/63
++ grep -f /tmp/tmp.oGaB4XvPf0/disabled_telemetry.version-service-cr.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.0-cw.log.json
+ desc 'operator VS should not have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
operator VS should not have telemetry
-----------------------------------------------------------------------------------

+ [[ -s /tmp/tmp.oGaB4XvPf0/disabled_telemetry.version-service.log.json ]]
+ desc 'telemetry was disabled in CR as well as in operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR as well as in operator
-----------------------------------------------------------------------------------

+ '[' 8.0-recommended == disabled -a disabled == disabled ']'
+ kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.YmmeU6U8OS
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZA26t1vd0o
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.YmmeU6U8OS
perconaxtradbcluster.pxc.percona.com/minimal-cluster patched
+ cat /tmp/tmp.ZA26t1vd0o
+ rm /tmp/tmp.YmmeU6U8OS /tmp/tmp.ZA26t1vd0o
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ head -1
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.adeWs8uML1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EbZZsYhxyn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.adeWs8uML1
++ cat /tmp/tmp.EbZZsYhxyn
++ rm /tmp/tmp.adeWs8uML1 /tmp/tmp.EbZZsYhxyn
++ return 0
+ kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-f5ddd9c4f-hfv55
++ mktemp
+ local LAST_OUT=/tmp/tmp.7nnz5Mtp7b
++ mktemp
+ local LAST_ERR=/tmp/tmp.kV8ClG7bSU
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-f5ddd9c4f-hfv55
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7nnz5Mtp7b
pod "percona-xtradb-cluster-operator-f5ddd9c4f-hfv55" deleted from pxc-operator namespace
+ cat /tmp/tmp.kV8ClG7bSU
+ rm /tmp/tmp.7nnz5Mtp7b /tmp/tmp.kV8ClG7bSU
+ return 0
+ kubectl_bin delete pxc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.IdI11VU2nj
++ mktemp
+ local LAST_ERR=/tmp/tmp.6BR1kqKXVB
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.IdI11VU2nj
perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.6BR1kqKXVB
+ rm /tmp/tmp.IdI11VU2nj /tmp/tmp.6BR1kqKXVB
+ return 0
+ kubectl_bin delete deploy pxc-client
++ mktemp
+ local LAST_OUT=/tmp/tmp.qlx7g0a7BP
++ mktemp
+ local LAST_ERR=/tmp/tmp.xxSs0axHFd
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete deploy pxc-client
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.qlx7g0a7BP
deployment.apps "pxc-client" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.xxSs0axHFd
+ rm /tmp/tmp.qlx7g0a7BP /tmp/tmp.xxSs0axHFd
+ return 0
+ sleep 30
+ kubectl_bin delete pod -l run=version-service-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.hohZ68yxx2
++ mktemp
+ local LAST_ERR=/tmp/tmp.F1yRIwO8ew
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -l run=version-service-cr
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.hohZ68yxx2
pod "version-service-cr-76d4659869-gssqx" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.F1yRIwO8ew
+ rm /tmp/tmp.hohZ68yxx2 /tmp/tmp.F1yRIwO8ew
+ return 0
+ kubectl_bin delete pod -l run=version-service
++ mktemp
+ local LAST_OUT=/tmp/tmp.UYMVDhKA8E
++ mktemp
+ local LAST_ERR=/tmp/tmp.y3MhsKqBg5
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -l run=version-service
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UYMVDhKA8E
pod "version-service-559c4df55c-q4hxf" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.y3MhsKqBg5
+ rm /tmp/tmp.UYMVDhKA8E /tmp/tmp.y3MhsKqBg5
+ return 0
+ check_telemetry_transfer http://version-service-cr.smart-update2-11189.svc.cluster.local:11000 disabled disabled
+ local cr_vs_uri=http://version-service-cr.smart-update2-11189.svc.cluster.local:11000
+ local cr_vs_channel=disabled
+ local telemetry_state=disabled
+ desc 'create PXC minimal cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC minimal cluster
-----------------------------------------------------------------------------------

+ cluster=minimal-cluster
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.hYd66Rh8d1
++ mktemp
+ local LAST_ERR=/tmp/tmp.NA4GSKFFmT
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.hYd66Rh8d1
deployment.apps/pxc-client created
+ cat /tmp/tmp.NA4GSKFFmT
+ rm /tmp/tmp.hYd66Rh8d1 /tmp/tmp.NA4GSKFFmT
+ return 0
+ yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"'
+ kubectl_bin apply -f -
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.FrJY0ISpzA
++ mktemp
+ local LAST_ERR=/tmp/tmp.Yq4GYcrOdL
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.FrJY0ISpzA
secret/minimal-cluster unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.Yq4GYcrOdL
+ rm /tmp/tmp.FrJY0ISpzA /tmp/tmp.Yq4GYcrOdL
+ return 0
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/deploy/cr-minimal.yaml
+ yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-11189.svc.cluster.local:11000"'
+ yq eval '.spec.logcollector.image="perconalab/fluentbit:main-logcollector"'
+ yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"'
+ kubectl_bin apply -f -
+ yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.41-32.1"'
+ yq eval '.spec.crVersion="9.9.9"'
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="disabled"'
+ yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe"'
+ local LAST_OUT=/tmp/tmp.mgMcouhWkf
++ mktemp
+ local LAST_ERR=/tmp/tmp.vvo4B7OGhO
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.mgMcouhWkf
perconaxtradbcluster.pxc.percona.com/minimal-cluster created
+ cat /tmp/tmp.vvo4B7OGhO
+ rm /tmp/tmp.mgMcouhWkf /tmp/tmp.vvo4B7OGhO
+ return 0
+ desc 'check if Pod is started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod is started
-----------------------------------------------------------------------------------

+ wait_for_running minimal-cluster-pxc 1
+ local name=minimal-cluster-pxc
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod minimal-cluster-pxc-0 480
+ local pod=minimal-cluster-pxc-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/minimal-cluster-pxc-0 condition met
waiting for pod/minimal-cluster-pxc-0 to become Ready.Ok
+ sleep 20
+ local proxy
++ get_proxy minimal-cluster
++ local target_cluster=minimal-cluster
+++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DZEdUIqNWn
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rH4x9ZfVo7
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.DZEdUIqNWn
+++ cat /tmp/tmp.rH4x9ZfVo7
+++ rm /tmp/tmp.DZEdUIqNWn /tmp/tmp.rH4x9ZfVo7
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo minimal-cluster-haproxy
++ return
+ proxy=minimal-cluster-haproxy
+ wait_for_running minimal-cluster-haproxy 1
+ local name=minimal-cluster-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod minimal-cluster-haproxy-0 480
+ local pod=minimal-cluster-haproxy-0
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo minimal-cluster-haproxy-0
+ local container=
+ set +o xtrace
pod/minimal-cluster-haproxy-0 condition met
waiting for pod/minimal-cluster-haproxy-0 to become Ready.Ok
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RFkN4myAFx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YWB8EoVv4g
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RFkN4myAFx
++ cat /tmp/tmp.YWB8EoVv4g
++ rm /tmp/tmp.RFkN4myAFx /tmp/tmp.YWB8EoVv4g
++ return 0
+ client_pod=pxc-client-67fc4995bb-zsjtj
+ wait_pod pxc-client-67fc4995bb-zsjtj
+ local pod=pxc-client-67fc4995bb-zsjtj
+ local max_retry=480
+ local ns=
++ echo pxc-client-67fc4995bb-zsjtj
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-zsjtj condition met
waiting for pod/pxc-client-67fc4995bb-zsjtj to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument.
command terminated with exit code 5
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FeekDkdHlC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1BOfdKyW3J
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FeekDkdHlC
++ cat /tmp/tmp.1BOfdKyW3J
++ rm /tmp/tmp.FeekDkdHlC /tmp/tmp.1BOfdKyW3J
++ return 0
+ client_pod=pxc-client-67fc4995bb-zsjtj
+ wait_pod pxc-client-67fc4995bb-zsjtj
+ local pod=pxc-client-67fc4995bb-zsjtj
+ local max_retry=480
+ local ns=
++ echo pxc-client-67fc4995bb-zsjtj
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-zsjtj condition met
waiting for pod/pxc-client-67fc4995bb-zsjtj to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument.
command terminated with exit code 5
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.time_ms")'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ grep -Eo '\{.*\}'
+ grep -E 'server request payload|unary call'
++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}'
+ kubectl_bin logs version-service-cr-76d4659869-fj97z
++ mktemp
+ local LAST_OUT=/tmp/tmp.qO8nZ3uGt5
++ mktemp
+ local LAST_ERR=/tmp/tmp.thHms4khQB
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs version-service-cr-76d4659869-fj97z
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.qO8nZ3uGt5
+ cat /tmp/tmp.thHms4khQB
+ rm /tmp/tmp.qO8nZ3uGt5 /tmp/tmp.thHms4khQB
+ return 0
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}'
+ grep -Eo '\{.*\}'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ jq 'del(."grpc.time_ms")'
+ grep -E 'server request payload|unary call'
+ kubectl_bin logs version-service-559c4df55c-nn8z2
++ mktemp
+ local LAST_OUT=/tmp/tmp.PccKGf0zQ5
++ mktemp
+ local LAST_ERR=/tmp/tmp.u4UjB4Jilr
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs version-service-559c4df55c-nn8z2
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.PccKGf0zQ5
+ cat /tmp/tmp.u4UjB4Jilr
+ rm /tmp/tmp.PccKGf0zQ5 /tmp/tmp.u4UjB4Jilr
+ return 0
+ local telemetry_log_file=disabled_telemetry.version-service-cw.log.json
+ desc 'telemetry was disabled in CR but in operator not'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR but in operator not
-----------------------------------------------------------------------------------

+ '[' disabled == disabled -a disabled == enabled ']'
+ local image_prefix=disabled
+ image_prefix=disabled
+ local telemetry_cr_log_file=disabled_telemetry.version-service-cr-disabled-cw.log.json
+ desc 'telemetry was disabled in operator but not in CR'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in operator but not in CR
-----------------------------------------------------------------------------------

+ [[ disabled == \d\i\s\a\b\l\e\d\-\r\e\c\o\m\m\e\n\d\e\d ]]
+ [[ disabled == \d\i\s\a\b\l\e\d\-\l\a\t\e\s\t ]]
+ desc 'telemetry was disabled in CR as well as in operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR as well as in operator
-----------------------------------------------------------------------------------

+ '[' disabled == disabled -a disabled == disabled ']'
+ desc 'CR VS should not have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
CR VS should not have telemetry
-----------------------------------------------------------------------------------

+ [[ -s /tmp/tmp.oGaB4XvPf0/disabled_telemetry.version-service-cr.log.json ]]
+ desc 'operator VS should not have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
operator VS should not have telemetry
-----------------------------------------------------------------------------------

+ [[ -s /tmp/tmp.oGaB4XvPf0/disabled_telemetry.version-service.log.json ]]
+ kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Y77FqX3dmn
++ mktemp
+ local LAST_ERR=/tmp/tmp.eeZhlo4NgR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Y77FqX3dmn
perconaxtradbcluster.pxc.percona.com/minimal-cluster patched
+ cat /tmp/tmp.eeZhlo4NgR
+ rm /tmp/tmp.Y77FqX3dmn /tmp/tmp.eeZhlo4NgR
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ head -1
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZNDpqTpgZM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hBHsFa3d9g
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZNDpqTpgZM
++ cat /tmp/tmp.hBHsFa3d9g
++ rm /tmp/tmp.ZNDpqTpgZM /tmp/tmp.hBHsFa3d9g
++ return 0
+ kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-f5ddd9c4f-phfxf
++ mktemp
+ local LAST_OUT=/tmp/tmp.yc4CFXjXEf
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZcOGY3dpix
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-f5ddd9c4f-phfxf
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.yc4CFXjXEf
pod "percona-xtradb-cluster-operator-f5ddd9c4f-phfxf" deleted from pxc-operator namespace
+ cat /tmp/tmp.ZcOGY3dpix
+ rm /tmp/tmp.yc4CFXjXEf /tmp/tmp.ZcOGY3dpix
+ return 0
+ kubectl_bin delete pxc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.phC92l6NEq
++ mktemp
+ local LAST_ERR=/tmp/tmp.8uRpCdTwLq
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.phC92l6NEq
perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.8uRpCdTwLq
+ rm /tmp/tmp.phC92l6NEq /tmp/tmp.8uRpCdTwLq
+ return 0
+ kubectl_bin delete deploy pxc-client
++ mktemp
+ local LAST_OUT=/tmp/tmp.kTtcKyftVb
++ mktemp
+ local LAST_ERR=/tmp/tmp.58xHD6zE2K
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete deploy pxc-client
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.kTtcKyftVb
deployment.apps "pxc-client" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.58xHD6zE2K
+ rm /tmp/tmp.kTtcKyftVb /tmp/tmp.58xHD6zE2K
+ return 0
+ sleep 30
+ kubectl_bin delete deployment version-service-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.1UeHRafBWp
++ mktemp
+ local LAST_ERR=/tmp/tmp.BuGF5cGB6M
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete deployment version-service-cr
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.1UeHRafBWp
deployment.apps "version-service-cr" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.BuGF5cGB6M
+ rm /tmp/tmp.1UeHRafBWp /tmp/tmp.BuGF5cGB6M
+ return 0
+ desc 'Telemetry testing finished'
+ set +o xtrace


-----------------------------------------------------------------------------------
Telemetry testing finished
-----------------------------------------------------------------------------------

+ desc 'PXC cluster with version service offline'
+ set +o xtrace


-----------------------------------------------------------------------------------
PXC cluster with version service offline
-----------------------------------------------------------------------------------

+ cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/smart-update-version-service-unreachable.yml /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe"' /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ spinup_pxc smart-update /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local cluster=smart-update
+ local config=/tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Pqay0eFVZS
++ mktemp
+ local LAST_ERR=/tmp/tmp.5Lgb4ypKt7
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Pqay0eFVZS
secret/my-cluster-secrets created
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.5Lgb4ypKt7
+ rm /tmp/tmp.Pqay0eFVZS /tmp/tmp.5Lgb4ypKt7
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ local pvc_name=
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ local pvc_name=
++ mktemp
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-11189~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ local LAST_OUT=/tmp/tmp.6uxMfEKmOL
++ mktemp
+ local LAST_ERR=/tmp/tmp.t0W80Gieze
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6uxMfEKmOL
deployment.apps/pxc-client created
+ cat /tmp/tmp.t0W80Gieze
+ rm /tmp/tmp.6uxMfEKmOL /tmp/tmp.t0W80Gieze
+ return 0
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ apply_config /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local config_file=/tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml ''
+ local input_file=/tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local pvc_name=
+ cat /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-11189~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
++ mktemp
+ local LAST_OUT=/tmp/tmp.1XCva4bxp5
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ local LAST_ERR=/tmp/tmp.U7QU2tkTcY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.1XCva4bxp5
perconaxtradbcluster.pxc.percona.com/smart-update created
+ cat /tmp/tmp.U7QU2tkTcY
+ rm /tmp/tmp.1XCva4bxp5 /tmp/tmp.U7QU2tkTcY
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.b1HOa7lmjs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ZTkNvfLV4m
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.b1HOa7lmjs
+++ cat /tmp/tmp.ZTkNvfLV4m
+++ rm /tmp/tmp.b1HOa7lmjs /tmp/tmp.ZTkNvfLV4m
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ local proxy=smart-update-haproxy
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-11189
++ mktemp
+ local LAST_OUT=/tmp/tmp.qVDrVUJUHA
++ mktemp
+ local LAST_ERR=/tmp/tmp.EkWtDfrGuv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-11189
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-11189
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-11189
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.qVDrVUJUHA
+ cat /tmp/tmp.EkWtDfrGuv
error: no matching resources found
+ rm /tmp/tmp.qVDrVUJUHA /tmp/tmp.EkWtDfrGuv
+ return 1
+ true
+ wait_for_running smart-update-haproxy 1
+ local name=smart-update-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-haproxy-0 480
+ local pod=smart-update-haproxy-0
+ local max_retry=480
+ local ns=
++ echo smart-update-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
Error from server (NotFound): pods "smart-update-haproxy-0" not found
waiting for pod/smart-update-haproxy-0 to become Ready.........................................Ok
+ wait_for_running smart-update-pxc 3
+ local name=smart-update-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-0 480
+ local pod=smart-update-pxc-0
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-0
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-0 condition met
waiting for pod/smart-update-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-1 480
+ local pod=smart-update-pxc-1
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-1 condition met
waiting for pod/smart-update-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-2 480
+ local pod=smart-update-pxc-2
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-2 condition met
waiting for pod/smart-update-pxc-2 to become Ready.Ok
+ sleep 10
++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UlrL70Z2GT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PcCR5k1fCw
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.UlrL70Z2GT
++ cat /tmp/tmp.PcCR5k1fCw
++ rm /tmp/tmp.UlrL70Z2GT /tmp/tmp.PcCR5k1fCw
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wW4WSW35Kr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q622tIEhQf
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wW4WSW35Kr
++ cat /tmp/tmp.q622tIEhQf
++ rm /tmp/tmp.wW4WSW35Kr /tmp/tmp.q622tIEhQf
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo pxc-client-6666d5df5c-pwvsv
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vhO20XSnTw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.U1bs2NIfjx
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vhO20XSnTw
++ cat /tmp/tmp.U1bs2NIfjx
++ rm /tmp/tmp.vhO20XSnTw /tmp/tmp.U1bs2NIfjx
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.64gQmpPFye
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XTkcR4FvYm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.64gQmpPFye
++ cat /tmp/tmp.XTkcR4FvYm
++ rm /tmp/tmp.64gQmpPFye /tmp/tmp.XTkcR4FvYm
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AkdoXyJmAH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VXvud03T8X
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AkdoXyJmAH
++ cat /tmp/tmp.VXvud03T8X
++ rm /tmp/tmp.AkdoXyJmAH /tmp/tmp.VXvud03T8X
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HV4gJVJPqM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7FZYRXTPe1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.HV4gJVJPqM
++ cat /tmp/tmp.7FZYRXTPe1
++ rm /tmp/tmp.HV4gJVJPqM /tmp/tmp.7FZYRXTPe1
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ is_keyring_plugin_in_use smart-update
+ local cluster=smart-update
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ kubectl exec smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
+ grep -E -o 'early-plugin-load=keyring_\w+.so'
+ return 1
+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=300
+ sleep 7
+ echo -n 'waiting for pxc/smart-update to be ready'
waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RVLaJ9YqLo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KgJrZCZEhz
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RVLaJ9YqLo
++ cat /tmp/tmp.KgJrZCZEhz
++ rm /tmp/tmp.RVLaJ9YqLo /tmp/tmp.KgJrZCZEhz
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WUtABPDB8i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AdFIVPCMDL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WUtABPDB8i
++ cat /tmp/tmp.AdFIVPCMDL
++ rm /tmp/tmp.WUtABPDB8i /tmp/tmp.AdFIVPCMDL
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.6NWp9XsRaK
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.NjENbFIHn7
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.6NWp9XsRaK
+++++ cat /tmp/tmp.NjENbFIHn7
+++++ rm /tmp/tmp.6NWp9XsRaK /tmp/tmp.NjENbFIHn7
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4NeDoeYaEQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d5Lb8lzAQN
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4NeDoeYaEQ
++ cat /tmp/tmp.d5Lb8lzAQN
++ rm /tmp/tmp.4NeDoeYaEQ /tmp/tmp.d5Lb8lzAQN
++ return 0
+ [[ 2 == \2 ]]
+ echo

++ kubectl_bin get pxc/smart-update -o 'jsonpath={.spec.pxc.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DeBwylqYm2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NM5V06P8zE
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc/smart-update -o 'jsonpath={.spec.pxc.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DeBwylqYm2
++ cat /tmp/tmp.NM5V06P8zE
++ rm /tmp/tmp.DeBwylqYm2 /tmp/tmp.NM5V06P8zE
++ return 0
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 != \p\e\r\c\o\n\a\/\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\:\8\.\0\.\4\1\-\3\2\.\1 ]]
+ desc 'PXC cluster update with recommended image by version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
PXC cluster update with recommended image by version service
-----------------------------------------------------------------------------------

+ vs_image=recommended
++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password'
++ local 'command=SELECT @@hostname hostname;'
++ local 'uri=-h smart-update-haproxy -uroot -proot_password'
+++ get_client_pod
+++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ezSce4GSjL
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1C5GgHv18T
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.ezSce4GSjL
+++ cat /tmp/tmp.1C5GgHv18T
+++ rm /tmp/tmp.ezSce4GSjL /tmp/tmp.1C5GgHv18T
+++ return 0
++ client_pod=pxc-client-6666d5df5c-pwvsv
++ wait_pod pxc-client-6666d5df5c-pwvsv
++ local pod=pxc-client-6666d5df5c-pwvsv
++ local max_retry=480
++ local ns=
+++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+++ echo pxc-client-6666d5df5c-pwvsv
+++ grep -E '^(pxc|proxysql)$'
++ local container=
++ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
++ set +o xtrace
+ initial_primary=smart-update-pxc-0
+ kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.QnAf0gdJy2
++ mktemp
+ local LAST_ERR=/tmp/tmp.OAk8omjCVZ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QnAf0gdJy2
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ cat /tmp/tmp.OAk8omjCVZ
+ rm /tmp/tmp.QnAf0gdJy2 /tmp/tmp.OAk8omjCVZ
+ return 0
+ sleep 55
+ check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ local cluster=smart-update
+ local initial_primary=smart-update-pxc-0
+ local pxc_size=3
+ local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ set +x
Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=300
+ sleep 7
+ echo -n 'waiting for pxc/smart-update to be ready'
waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2LGQ6e0wsb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5dKCg60TJq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2LGQ6e0wsb
++ cat /tmp/tmp.5dKCg60TJq
++ rm /tmp/tmp.2LGQ6e0wsb /tmp/tmp.5dKCg60TJq
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L5EVLwpgpa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gBjvFH8aZy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.L5EVLwpgpa
++ cat /tmp/tmp.gBjvFH8aZy
++ rm /tmp/tmp.L5EVLwpgpa /tmp/tmp.gBjvFH8aZy
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.Wtg3krsSlU
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.Up73rxrIzU
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.Wtg3krsSlU
+++++ cat /tmp/tmp.Up73rxrIzU
+++++ rm /tmp/tmp.Wtg3krsSlU /tmp/tmp.Up73rxrIzU
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vTsf57nZKg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VmbezoWSwl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vTsf57nZKg
++ cat /tmp/tmp.VmbezoWSwl
++ rm /tmp/tmp.vTsf57nZKg /tmp/tmp.VmbezoWSwl
++ return 0
+ [[ 2 == \2 ]]
+ echo

++ seq 0 2
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FhitcAL2ze
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XVlsAHNPe1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FhitcAL2ze
++ cat /tmp/tmp.XVlsAHNPe1
++ rm /tmp/tmp.FhitcAL2ze /tmp/tmp.XVlsAHNPe1
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FEsGWQX8LU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MPDT2L6ECn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FEsGWQX8LU
++ cat /tmp/tmp.MPDT2L6ECn
++ rm /tmp/tmp.FEsGWQX8LU /tmp/tmp.MPDT2L6ECn
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Dx4oJi6bzu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VbgrEk0h0c
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Dx4oJi6bzu
++ cat /tmp/tmp.VbgrEk0h0c
++ rm /tmp/tmp.Dx4oJi6bzu /tmp/tmp.VbgrEk0h0c
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ kubectl_bin delete -f /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.EF6jTDycOn
++ mktemp
+ local LAST_ERR=/tmp/tmp.gqBzwYvOJt
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EF6jTDycOn
perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.gqBzwYvOJt
+ rm /tmp/tmp.EF6jTDycOn /tmp/tmp.gqBzwYvOJt
+ return 0
+ kubectl_bin delete pvc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.7EAscko8VD
++ mktemp
+ local LAST_ERR=/tmp/tmp.7mezjLTQST
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pvc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7EAscko8VD
persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-11189 namespace
persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-11189 namespace
persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.7mezjLTQST
+ rm /tmp/tmp.7EAscko8VD /tmp/tmp.7mezjLTQST
+ return 0
+ desc 'PXC cluster update with the latest image by version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
PXC cluster update with the latest image by version service
-----------------------------------------------------------------------------------

+ spinup_pxc smart-update /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local cluster=smart-update
+ local config=/tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.XCO1eSnCAg
++ mktemp
+ local LAST_ERR=/tmp/tmp.i8non8lK6C
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XCO1eSnCAg
secret/my-cluster-secrets unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.i8non8lK6C
+ rm /tmp/tmp.XCO1eSnCAg /tmp/tmp.i8non8lK6C
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ local pvc_name=
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-11189~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ipBIoudFoG
++ mktemp
+ local LAST_ERR=/tmp/tmp.63ONauQ9Js
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ipBIoudFoG
deployment.apps/pxc-client unchanged
+ cat /tmp/tmp.63ONauQ9Js
+ rm /tmp/tmp.ipBIoudFoG /tmp/tmp.63ONauQ9Js
+ return 0
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ apply_config /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local config_file=/tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml ''
+ local input_file=/tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local pvc_name=
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ cat /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-11189~
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#'
+ local LAST_OUT=/tmp/tmp.hLjqA4WVZO
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
++ mktemp
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ local LAST_ERR=/tmp/tmp.hweN5OQ8YE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.hLjqA4WVZO
perconaxtradbcluster.pxc.percona.com/smart-update created
+ cat /tmp/tmp.hweN5OQ8YE
+ rm /tmp/tmp.hLjqA4WVZO /tmp/tmp.hweN5OQ8YE
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.suWIcOYlLO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ZJwExbkPzj
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.suWIcOYlLO
+++ cat /tmp/tmp.ZJwExbkPzj
+++ rm /tmp/tmp.suWIcOYlLO /tmp/tmp.ZJwExbkPzj
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ local proxy=smart-update-haproxy
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-11189
++ mktemp
+ local LAST_OUT=/tmp/tmp.WfvQRL9aKQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.nb6xfQFIiH
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-11189
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-11189
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-11189
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.WfvQRL9aKQ
+ cat /tmp/tmp.nb6xfQFIiH
error: no matching resources found
+ rm /tmp/tmp.WfvQRL9aKQ /tmp/tmp.nb6xfQFIiH
+ return 1
+ true
+ wait_for_running smart-update-haproxy 1
+ local name=smart-update-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-haproxy-0 480
+ local pod=smart-update-haproxy-0
+ local max_retry=480
+ local ns=
++ echo smart-update-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/smart-update-haproxy-0 condition met
waiting for pod/smart-update-haproxy-0 to become Ready.Ok
+ wait_for_running smart-update-pxc 3
+ local name=smart-update-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-0 480
+ local pod=smart-update-pxc-0
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-0 condition met
waiting for pod/smart-update-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-1 480
+ local pod=smart-update-pxc-1
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-1 condition met
waiting for pod/smart-update-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-2 480
+ local pod=smart-update-pxc-2
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-2 condition met
waiting for pod/smart-update-pxc-2 to become Ready.Ok
+ sleep 10
++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eP0OM7W8h5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Rh7oY9mcyh
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eP0OM7W8h5
++ cat /tmp/tmp.Rh7oY9mcyh
++ rm /tmp/tmp.eP0OM7W8h5 /tmp/tmp.Rh7oY9mcyh
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.j3AbOhbVvO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tYIJfGAJpZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.j3AbOhbVvO
++ cat /tmp/tmp.tYIJfGAJpZ
++ rm /tmp/tmp.j3AbOhbVvO /tmp/tmp.tYIJfGAJpZ
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xLbdyEfsvA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RSvsYKmhpR
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xLbdyEfsvA
++ cat /tmp/tmp.RSvsYKmhpR
++ rm /tmp/tmp.xLbdyEfsvA /tmp/tmp.RSvsYKmhpR
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.d44RRTgbxC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yhG4GBVCF7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.d44RRTgbxC
++ cat /tmp/tmp.yhG4GBVCF7
++ rm /tmp/tmp.d44RRTgbxC /tmp/tmp.yhG4GBVCF7
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.x3LztLzZUK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.709hQmUizD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.x3LztLzZUK
++ cat /tmp/tmp.709hQmUizD
++ rm /tmp/tmp.x3LztLzZUK /tmp/tmp.709hQmUizD
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c8Sp6EtviI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2ycPK7A0Hj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.c8Sp6EtviI
++ cat /tmp/tmp.2ycPK7A0Hj
++ rm /tmp/tmp.c8Sp6EtviI /tmp/tmp.2ycPK7A0Hj
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ is_keyring_plugin_in_use smart-update
+ local cluster=smart-update
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ kubectl exec smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
+ grep -E -o 'early-plugin-load=keyring_\w+.so'
+ return 1
+ vs_image=latest
++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password'
++ local 'command=SELECT @@hostname hostname;'
++ local 'uri=-h smart-update-haproxy -uroot -proot_password'
+++ get_client_pod
+++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.xKCmOdOeTj
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.GrgF1Vf9Ft
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.xKCmOdOeTj
+++ cat /tmp/tmp.GrgF1Vf9Ft
+++ rm /tmp/tmp.xKCmOdOeTj /tmp/tmp.GrgF1Vf9Ft
+++ return 0
++ client_pod=pxc-client-6666d5df5c-pwvsv
++ wait_pod pxc-client-6666d5df5c-pwvsv
++ local pod=pxc-client-6666d5df5c-pwvsv
++ local max_retry=480
++ local ns=
+++ echo pxc-client-6666d5df5c-pwvsv
+++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+++ grep -E '^(pxc|proxysql)$'
++ local container=
++ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
++ set +o xtrace
+ initial_primary=smart-update-pxc-0
+ kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.QeBCa4BAi4
++ mktemp
+ local LAST_ERR=/tmp/tmp.dtWSijvdmg
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QeBCa4BAi4
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ cat /tmp/tmp.dtWSijvdmg
+ rm /tmp/tmp.QeBCa4BAi4 /tmp/tmp.dtWSijvdmg
+ return 0
+ sleep 55
+ check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ local cluster=smart-update
+ local initial_primary=smart-update-pxc-0
+ local pxc_size=3
+ local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ set +x
Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=300
+ sleep 7
+ echo -n 'waiting for pxc/smart-update to be ready'
waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4vsnrfl4dG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6IFeEnF5kI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4vsnrfl4dG
++ cat /tmp/tmp.6IFeEnF5kI
++ rm /tmp/tmp.4vsnrfl4dG /tmp/tmp.6IFeEnF5kI
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BQZgldetCV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UXpwBw8inU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BQZgldetCV
++ cat /tmp/tmp.UXpwBw8inU
++ rm /tmp/tmp.BQZgldetCV /tmp/tmp.UXpwBw8inU
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.UiAU8UZVSk
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.JPZzy2ROVs
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.UiAU8UZVSk
+++++ cat /tmp/tmp.JPZzy2ROVs
+++++ rm /tmp/tmp.UiAU8UZVSk /tmp/tmp.JPZzy2ROVs
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iKTxhNYWEC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4aAO6vxvjQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.iKTxhNYWEC
++ cat /tmp/tmp.4aAO6vxvjQ
++ rm /tmp/tmp.iKTxhNYWEC /tmp/tmp.4aAO6vxvjQ
++ return 0
+ [[ 2 == \2 ]]
+ echo

++ seq 0 2
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CYfQXwPfoZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Waoc4wMunn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.CYfQXwPfoZ
++ cat /tmp/tmp.Waoc4wMunn
++ rm /tmp/tmp.CYfQXwPfoZ /tmp/tmp.Waoc4wMunn
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2CQTVhn7Fj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QJH4xVm4he
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2CQTVhn7Fj
++ cat /tmp/tmp.QJH4xVm4he
++ rm /tmp/tmp.2CQTVhn7Fj /tmp/tmp.QJH4xVm4he
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aVUTjx8811
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m0OOXE4aGM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.aVUTjx8811
++ cat /tmp/tmp.m0OOXE4aGM
++ rm /tmp/tmp.aVUTjx8811 /tmp/tmp.m0OOXE4aGM
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-6666d5df5c-pwvsv
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ kubectl_bin delete -f /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.4EV0K3Zs73
++ mktemp
+ local LAST_ERR=/tmp/tmp.G128Q3ltcX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.4EV0K3Zs73
perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.G128Q3ltcX
+ rm /tmp/tmp.4EV0K3Zs73 /tmp/tmp.G128Q3ltcX
+ return 0
+ kubectl_bin delete pvc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.EQpLyANMb1
++ mktemp
+ local LAST_ERR=/tmp/tmp.7XLx3P9MxY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pvc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EQpLyANMb1
persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-11189 namespace
persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-11189 namespace
persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.7XLx3P9MxY
+ rm /tmp/tmp.EQpLyANMb1 /tmp/tmp.7XLx3P9MxY
+ return 0
+ desc 'PXC cluster update with explicitly specified image inside version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
PXC cluster update with explicitly specified image inside version service
-----------------------------------------------------------------------------------

+ spinup_pxc smart-update /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local cluster=smart-update
+ local config=/tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.JcsUXD3g5R
++ mktemp
+ local LAST_ERR=/tmp/tmp.NlibtWKVV6
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.JcsUXD3g5R
secret/my-cluster-secrets unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.NlibtWKVV6
+ rm /tmp/tmp.JcsUXD3g5R /tmp/tmp.NlibtWKVV6
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ local pvc_name=
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
+ local pvc_name=
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/conf/client.yml
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#'
+ local LAST_OUT=/tmp/tmp.75zMJc1b4K
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
++ mktemp
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-11189~
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ local LAST_ERR=/tmp/tmp.pDK3hLQYOe
+ local exit_status=0
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
++ seq 0 2
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.75zMJc1b4K
deployment.apps/pxc-client unchanged
+ cat /tmp/tmp.pDK3hLQYOe
+ rm /tmp/tmp.75zMJc1b4K /tmp/tmp.pDK3hLQYOe
+ return 0
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ apply_config /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local config_file=/tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml ''
+ local input_file=/tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ local pvc_name=
+ kubectl_bin apply -f -
+ cat /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Alq03DILk8
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2434-3b65d7fe#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.41-32.1#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-11189~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ local LAST_ERR=/tmp/tmp.2KYzn4snvy
+ local exit_status=0
++ seq 0 2
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.41-32.1#'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Alq03DILk8
perconaxtradbcluster.pxc.percona.com/smart-update created
+ cat /tmp/tmp.2KYzn4snvy
+ rm /tmp/tmp.Alq03DILk8 /tmp/tmp.2KYzn4snvy
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.vy8cKmR57g
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.th1wnyYcrg
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.vy8cKmR57g
+++ cat /tmp/tmp.th1wnyYcrg
+++ rm /tmp/tmp.vy8cKmR57g /tmp/tmp.th1wnyYcrg
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ local proxy=smart-update-haproxy
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-11189
++ mktemp
+ local LAST_OUT=/tmp/tmp.TqT39qd856
++ mktemp
+ local LAST_ERR=/tmp/tmp.yMM9Fi45wN
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-11189
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-11189
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-11189
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.TqT39qd856
+ cat /tmp/tmp.yMM9Fi45wN
error: no matching resources found
+ rm /tmp/tmp.TqT39qd856 /tmp/tmp.yMM9Fi45wN
+ return 1
+ true
+ wait_for_running smart-update-haproxy 1
+ local name=smart-update-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-haproxy-0 480
+ local pod=smart-update-haproxy-0
+ local max_retry=480
+ local ns=
++ echo smart-update-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/smart-update-haproxy-0 condition met
waiting for pod/smart-update-haproxy-0 to become Ready.Ok
+ wait_for_running smart-update-pxc 3
+ local name=smart-update-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-0 480
+ local pod=smart-update-pxc-0
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-0 condition met
waiting for pod/smart-update-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-1 480
+ local pod=smart-update-pxc-1
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-1 condition met
waiting for pod/smart-update-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-2 480
+ local pod=smart-update-pxc-2
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-2 condition met
waiting for pod/smart-update-pxc-2 to become Ready.Ok
+ sleep 10
++ kubectl get pxc smart-update -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1j3uHPdFP5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v1R1Uon96x
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1j3uHPdFP5
++ cat /tmp/tmp.v1R1Uon96x
++ rm /tmp/tmp.1j3uHPdFP5 /tmp/tmp.v1R1Uon96x
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q7x8CpNAXP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KqrVqbfRhp
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.q7x8CpNAXP
++ cat /tmp/tmp.KqrVqbfRhp
++ rm /tmp/tmp.q7x8CpNAXP /tmp/tmp.KqrVqbfRhp
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h smart-update-haproxy -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HigCBvcdK2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rh3DGotOHC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.HigCBvcdK2
++ cat /tmp/tmp.rh3DGotOHC
++ rm /tmp/tmp.HigCBvcdK2 /tmp/tmp.rh3DGotOHC
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kOEtCYl1pM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6tSzMlvvUO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.kOEtCYl1pM
++ cat /tmp/tmp.6tSzMlvvUO
++ rm /tmp/tmp.kOEtCYl1pM /tmp/tmp.6tSzMlvvUO
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8Rv0BZOqeT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OV3H1577ng
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8Rv0BZOqeT
++ cat /tmp/tmp.OV3H1577ng
++ rm /tmp/tmp.8Rv0BZOqeT /tmp/tmp.OV3H1577ng
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WbhHODAknA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cEp59o9sIl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WbhHODAknA
++ cat /tmp/tmp.cEp59o9sIl
++ rm /tmp/tmp.WbhHODAknA /tmp/tmp.cEp59o9sIl
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ is_keyring_plugin_in_use smart-update
+ local cluster=smart-update
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ kubectl exec smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
+ grep -E -o 'early-plugin-load=keyring_\w+.so'
+ return 1
++ head -n1
++ grep :8.0
++ tail -n2
++ jq -r '.versions[].matrix.pxc[].imagePath'
++ sort -V
+++ get_operator_pod
+++ local label_prefix=app.kubernetes.io/
++++ grep -c percona-xtradb-cluster-operator
++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ local check_label=1
+++ [[ 1 -eq 0 ]]
+++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
+++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ head -1
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.n54OkzuiRP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.BZGRkumunJ
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.n54OkzuiRP
+++ cat /tmp/tmp.BZGRkumunJ
+++ rm /tmp/tmp.n54OkzuiRP /tmp/tmp.BZGRkumunJ
+++ return 0
++ kubectl_bin exec -ti percona-xtradb-cluster-operator-f5ddd9c4f-ww4xk -n pxc-operator -- curl -s http://version-service.smart-update2-11189.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WtW6oSJeaz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RczDRRa1Ns
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -ti percona-xtradb-cluster-operator-f5ddd9c4f-ww4xk -n pxc-operator -- curl -s http://version-service.smart-update2-11189.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WtW6oSJeaz
++ cat /tmp/tmp.RczDRRa1Ns
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.WtW6oSJeaz /tmp/tmp.RczDRRa1Ns
++ return 0
+ vs_image=percona/percona-xtradb-cluster:8.0.41-32.1
++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password'
++ local 'command=SELECT @@hostname hostname;'
++ local 'uri=-h smart-update-haproxy -uroot -proot_password'
+++ get_client_pod
+++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DE2vH8YYHr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5VKPIor6hq
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.DE2vH8YYHr
+++ cat /tmp/tmp.5VKPIor6hq
+++ rm /tmp/tmp.DE2vH8YYHr /tmp/tmp.5VKPIor6hq
+++ return 0
++ client_pod=pxc-client-6666d5df5c-pwvsv
++ wait_pod pxc-client-6666d5df5c-pwvsv
++ local pod=pxc-client-6666d5df5c-pwvsv
++ local max_retry=480
++ local ns=
+++ echo pxc-client-6666d5df5c-pwvsv
+++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+++ grep -E '^(pxc|proxysql)$'
++ local container=
++ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
++ set +o xtrace
+ initial_primary=smart-update-pxc-0
+ kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.41-32.1","schedule": "* * * * *"}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.6d9lI0V5aN
++ mktemp
+ local LAST_ERR=/tmp/tmp.07Dv80Bfsl
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.41-32.1","schedule": "* * * * *"}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6d9lI0V5aN
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ cat /tmp/tmp.07Dv80Bfsl
+ rm /tmp/tmp.6d9lI0V5aN /tmp/tmp.07Dv80Bfsl
+ return 0
+ sleep 55
+ check_last_pod_to_update smart-update smart-update-pxc-0 3 percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.41-32.1
+ local cluster=smart-update
+ local initial_primary=smart-update-pxc-0
+ local pxc_size=3
+ local target_image=percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.41-32.1
+ set +x
Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=300
+ sleep 7
+ echo -n 'waiting for pxc/smart-update to be ready'
waiting for pxc/smart-update to be ready++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1IPnlxo8dF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.W00szuXd3r
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1IPnlxo8dF
++ cat /tmp/tmp.W00szuXd3r
++ rm /tmp/tmp.1IPnlxo8dF /tmp/tmp.W00szuXd3r
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IyeShRetGO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.a7zU3uInip
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IyeShRetGO
++ cat /tmp/tmp.a7zU3uInip
++ rm /tmp/tmp.IyeShRetGO /tmp/tmp.a7zU3uInip
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.iIoMyrGCud
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.PQ3NbKH9zQ
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.iIoMyrGCud
+++++ cat /tmp/tmp.PQ3NbKH9zQ
+++++ rm /tmp/tmp.iIoMyrGCud /tmp/tmp.PQ3NbKH9zQ
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b1rJngpmeP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uKF2RcM8rK
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.b1rJngpmeP
++ cat /tmp/tmp.uKF2RcM8rK
++ rm /tmp/tmp.b1rJngpmeP /tmp/tmp.uKF2RcM8rK
++ return 0
+ [[ 2 == \2 ]]
+ echo

++ seq 0 2
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.enAA6tYq6v
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tnwtqkn24M
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.enAA6tYq6v
++ cat /tmp/tmp.tnwtqkn24M
++ rm /tmp/tmp.enAA6tYq6v /tmp/tmp.tnwtqkn24M
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tvMSZFVxbT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XtqpVtYAxm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.tvMSZFVxbT
++ cat /tmp/tmp.XtqpVtYAxm
++ rm /tmp/tmp.tvMSZFVxbT /tmp/tmp.XtqpVtYAxm
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo pxc-client-6666d5df5c-pwvsv
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.4 ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1-80.sql ]]
+ [[ percona/percona-xtradb-cluster:8.0.41-32.1 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Z3pzjQZ429
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sVvfcmk9NW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Z3pzjQZ429
++ cat /tmp/tmp.sVvfcmk9NW
++ rm /tmp/tmp.Z3pzjQZ429 /tmp/tmp.sVvfcmk9NW
++ return 0
+ client_pod=pxc-client-6666d5df5c-pwvsv
+ wait_pod pxc-client-6666d5df5c-pwvsv
+ local pod=pxc-client-6666d5df5c-pwvsv
+ local max_retry=480
+ local ns=
++ echo pxc-client-6666d5df5c-pwvsv
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6666d5df5c-pwvsv condition met
waiting for pod/pxc-client-6666d5df5c-pwvsv to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.oGaB4XvPf0/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.oGaB4XvPf0/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.oGaB4XvPf0/select-1.sql
+ kubectl_bin delete -f /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.nolgnlWrm9
++ mktemp
+ local LAST_ERR=/tmp/tmp.G9IvUvPLf2
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /tmp/tmp.oGaB4XvPf0/smart-update-version-service-unreachable.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nolgnlWrm9
perconaxtradbcluster.pxc.percona.com "smart-update" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.G9IvUvPLf2
+ rm /tmp/tmp.nolgnlWrm9 /tmp/tmp.G9IvUvPLf2
+ return 0
+ kubectl_bin delete pvc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.wHPGMFIuUy
++ mktemp
+ local LAST_ERR=/tmp/tmp.TC1O8ImdT1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pvc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.wHPGMFIuUy
persistentvolumeclaim "datadir-smart-update-pxc-0" deleted from smart-update2-11189 namespace
persistentvolumeclaim "datadir-smart-update-pxc-1" deleted from smart-update2-11189 namespace
persistentvolumeclaim "datadir-smart-update-pxc-2" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.TC1O8ImdT1
+ rm /tmp/tmp.wHPGMFIuUy /tmp/tmp.TC1O8ImdT1
+ return 0
+ desc cleanup
+ set +o xtrace


-----------------------------------------------------------------------------------
cleanup
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/vs.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.GTWg6gc9AU
++ mktemp
+ local LAST_ERR=/tmp/tmp.iDN9asrBu8
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2434/e2e-tests/smart-update2/conf/vs.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.GTWg6gc9AU
deployment.apps "version-service" deleted from smart-update2-11189 namespace
service "version-service" deleted from smart-update2-11189 namespace
+ cat /tmp/tmp.iDN9asrBu8
+ rm /tmp/tmp.GTWg6gc9AU /tmp/tmp.iDN9asrBu8
+ return 0
+ destroy smart-update2-11189
+ local namespace=smart-update2-11189
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+ grep -v 'get backup status: Job.batch'
+ sort -u
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ tee /tmp/tmp.oGaB4XvPf0/operator.log
+ grep -v 'the object has been modified'
+ grep -v level=info
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ head -1
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VMXcmvGPOq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZRRUoG1KII
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VMXcmvGPOq
++ cat /tmp/tmp.ZRRUoG1KII
++ rm /tmp/tmp.VMXcmvGPOq /tmp/tmp.ZRRUoG1KII
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-f5ddd9c4f-ww4xk
++ mktemp
+ local LAST_OUT=/tmp/tmp.WsLPTDa9ol
++ mktemp
+ local LAST_ERR=/tmp/tmp.EODx3fL5Gz
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-f5ddd9c4f-ww4xk
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.WsLPTDa9ol
+ cat /tmp/tmp.EODx3fL5Gz
+ rm /tmp/tmp.WsLPTDa9ol /tmp/tmp.EODx3fL5Gz
+ return 0
2026/04/09 04:28:57 http: TLS handshake error from 10.8.217.11:60298: EOF
2026/04/09 04:28:57 http: TLS handshake error from 10.8.217.4:42182: EOF
2026-04-09T04:28:35.893Z	INFO	setup	Feature gates	{"PXCO_FEATURE_GATES": "", "enabled": ""}
2026-04-09T04:28:35.893Z	INFO	setup	Manager starting up	{"gitCommit": "3b65d7fe6a0a5db845238d34252d90db1f886e1d", "gitBranch": "PR-2434-3b65d7fe", "buildTime": "2026-04-09T01:11:10Z", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"}
2026-04-09T04:28:35.893Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.32.13-gke.1205000"}
2026-04-09T04:28:35.897Z	INFO	setup	Registering Components.
2026-04-09T04:28:36.278Z	INFO	controller-runtime.certwatcher	Starting certificate poll+watcher	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"}
2026-04-09T04:28:36.278Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"}
2026-04-09T04:28:36.278Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2026-04-09T04:28:36.278Z	INFO	controller-runtime.metrics	Starting metrics server
2026-04-09T04:28:36.278Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2026-04-09T04:28:36.278Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2026-04-09T04:28:36.278Z	INFO	controller-runtime.webhook	Starting webhook server
2026-04-09T04:28:36.278Z	INFO	setup	Starting the Cmd.
2026-04-09T04:28:36.278Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2026-04-09T04:28:36.379Z	INFO	Attempting to acquire leader lease...	{"lock": "pxc-operator/08db1feb.percona.com"}
2026-04-09T04:28:56.702Z	DEBUG	events	percona-xtradb-cluster-operator-f5ddd9c4f-ww4xk_3f15b9fe-97d0-47ed-bde3-7fdfd16a2199 became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"8a770347-4a1d-4371-b681-1112471ee423","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1775708936696399009"}, "reason": "LeaderElection"}
2026-04-09T04:28:56.702Z	INFO	Successfully acquired lease	{"lock": "pxc-operator/08db1feb.percona.com"}
2026-04-09T04:28:56.703Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2026-04-09T04:28:56.703Z	INFO	Starting EventSource	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"}
2026-04-09T04:28:56.703Z	INFO	Starting EventSource	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"}
2026-04-09T04:28:56.703Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2026-04-09T04:28:56.803Z	INFO	Starting Controller	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"}
2026-04-09T04:28:56.803Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1}
2026-04-09T04:28:56.904Z	INFO	Starting Controller	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"}
2026-04-09T04:28:56.904Z	INFO	Starting Controller	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"}
2026-04-09T04:28:56.904Z	INFO	Starting workers	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1}
2026-04-09T04:28:56.904Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1}
2026-04-09T04:29:01.886Z	ERROR	Reconciler error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"minimal-cluster","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "minimal-cluster", "reconcileID": "8f930cd9-f658-4f3f-97ba-8446b2d16a4c", "error": "failed to update cr finalizers: Internal error occurred: failed calling webhook \"validationwebhook.pxc.percona.com\": failed to call webhook: Post \"https://percona-xtradb-cluster-operator.pxc-operator.svc:443/validate-percona-xtradbcluster?timeout=10s\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \"crypto/rsa: verification error\" while trying to verify candidate authority certificate \"Root CA\")", "errorVerbose": "Internal error occurred: failed calling webhook \"validationwebhook.pxc.percona.com\": failed to call webhook: Post \"https://percona-xtradb-cluster-operator.pxc-operator.svc:443/validate-percona-xtradbcluster?timeout=10s\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \"crypto/rsa: verification error\" while trying to verify candidate authority certificate \"Root CA\")\nfailed to update cr finalizers\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2026-04-09T04:29:01.886Z	ERROR	Update status	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"minimal-cluster","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "minimal-cluster", "reconcileID": "8f930cd9-f658-4f3f-97ba-8446b2d16a4c", "error": "conditions are not equal: expected [{disabled tls 2026-04-09 04:26:20 +0000 UTC  } {True initializing 2026-04-09 04:26:21 +0000 UTC  } {True ready 2026-04-09 04:27:49 +0000 UTC  } {True error 2026-04-09 04:28:57 +0000 UTC ErrorReconcile Internal error occurred: failed calling webhook \"validationwebhook.pxc.percona.com\": failed to call webhook: Post \"https://percona-xtradb-cluster-operator.pxc-operator.svc:443/validate-percona-xtradbcluster?timeout=10s\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \"crypto/rsa: verification error\" while trying to verify candidate authority certificate \"Root CA\")}], have [{disabled tls 2026-04-09 04:26:20 +0000 UTC  } {True initializing 2026-04-09 04:26:21 +0000 UTC  } {True ready 2026-04-09 04:27:49 +0000 UTC  }]", "errorVerbose": "conditions are not equal: expected [{disabled tls 2026-04-09 04:26:20 +0000 UTC  } {True initializing 2026-04-09 04:26:21 +0000 UTC  } {True ready 2026-04-09 04:27:49 +0000 UTC  } {True error 2026-04-09 04:28:57 +0000 UTC ErrorReconcile Internal error occurred: failed calling webhook \"validationwebhook.pxc.percona.com\": failed to call webhook: Post \"https://percona-xtradb-cluster-operator.pxc-operator.svc:443/validate-percona-xtradbcluster?timeout=10s\": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \"crypto/rsa: verification error\" while trying to verify candidate authority certificate \"Root CA\")}], have [{disabled tls 2026-04-09 04:26:20 +0000 UTC  } {True initializing 2026-04-09 04:26:21 +0000 UTC  } {True ready 2026-04-09 04:27:49 +0000 UTC  }]\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus.func3\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:176\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.35.3/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.35.3/util/retry/util.go:50\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:169\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:45\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:257\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:484\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2026-04-09T04:29:06.521Z	ERROR	Update status	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"minimal-cluster","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "minimal-cluster", "reconcileID": "2dd09937-5054-4ce7-a55c-2d9d5a04d0e6", "error": "PerconaXtraDBCluster.pxc.percona.com \"minimal-cluster\" not found"}
2026-04-09T04:29:41.369Z	INFO	spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "b10efe68-2bb8-4286-93d8-7c4e44a341cb"}
2026-04-09T04:29:44.446Z	INFO	spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "b10efe68-2bb8-4286-93d8-7c4e44a341cb"}
2026-04-09T04:29:44.509Z	INFO	spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "b10efe68-2bb8-4286-93d8-7c4e44a341cb"}
2026-04-09T04:29:47.610Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "b10efe68-2bb8-4286-93d8-7c4e44a341cb", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-04-09T04:29:47.634Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "b10efe68-2bb8-4286-93d8-7c4e44a341cb", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-04-09T04:29:48.193Z	ERROR	Reconciler error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "b10efe68-2bb8-4286-93d8-7c4e44a341cb", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:92\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:25\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:27\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:50\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:543\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:556\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2026-04-09T04:29:48.312Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "da897612-15e4-4ee0-a72f-cbe5172d968d", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-04-09T04:29:48.392Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "da897612-15e4-4ee0-a72f-cbe5172d968d", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-04-09T04:29:48.443Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "da897612-15e4-4ee0-a72f-cbe5172d968d", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T04:29:48.482Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "da897612-15e4-4ee0-a72f-cbe5172d968d", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T04:29:48.545Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "da897612-15e4-4ee0-a72f-cbe5172d968d", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T04:29:48.663Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "da897612-15e4-4ee0-a72f-cbe5172d968d", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T04:29:49.736Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "5c596582-f737-4e70-be0b-547ea0191224", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-04-09T04:29:49.785Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "5c596582-f737-4e70-be0b-547ea0191224", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-04-09T04:31:16.606Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "e59bb34a-8afe-4f09-9bf2-b7e9f1d2f09c", "user": "operator"}
2026-04-09T04:31:16.649Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "e59bb34a-8afe-4f09-9bf2-b7e9f1d2f09c", "user": "monitor"}
2026-04-09T04:31:16.718Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "e59bb34a-8afe-4f09-9bf2-b7e9f1d2f09c"}
2026-04-09T04:31:16.752Z	INFO	monitor user privileges granted	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "e59bb34a-8afe-4f09-9bf2-b7e9f1d2f09c"}
2026-04-09T04:31:16.788Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "e59bb34a-8afe-4f09-9bf2-b7e9f1d2f09c", "user": "xtrabackup"}
2026-04-09T04:31:16.835Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "e59bb34a-8afe-4f09-9bf2-b7e9f1d2f09c"}
2026-04-09T04:31:16.869Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "e59bb34a-8afe-4f09-9bf2-b7e9f1d2f09c", "user": "replication"}
2026-04-09T04:33:54.113Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "56b4f1c8-4e90-42ee-be1b-b420457373b8", "user": "root"}
2026-04-09T04:33:55.229Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "56b4f1c8-4e90-42ee-be1b-b420457373b8", "new version": "8.0.41-32.1"}
2026-04-09T04:35:35.752Z	INFO	add new job	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "19a5aa91-3c82-43cc-9ac4-3983aa170bec", "name": "ensure-version/smart-update2-11189/smart-update", "schedule": "* * * * *"}
2026-04-09T04:36:35.000Z	DEBUG	Use version service endpoint	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "19a5aa91-3c82-43cc-9ac4-3983aa170bec", "endpoint": "http://version-service.smart-update2-11189.svc.cluster.local:11000"}
2026-04-09T04:36:35.051Z	ERROR	failed to ensure version	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "19a5aa91-3c82-43cc-9ac4-3983aa170bec", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=f0055933-c2fb-481e-8bf7-39b1f1c33913&databaseVersion=8.0.41-32.1&kubeVersion=v1.32.13-gke.1205000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=f0055933-c2fb-481e-8bf7-39b1f1c33913&databaseVersion=8.0.41-32.1&kubeVersion=v1.32.13-gke.1205000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:300\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:302\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2026-04-09T04:37:32.922Z	ERROR	Update status	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "d8acf1e4-dbe7-477c-ad52-37ed2828c7c1", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"}
2026-04-09T04:37:35.001Z	INFO	cluster is not found, deleting the job	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "19a5aa91-3c82-43cc-9ac4-3983aa170bec", "name": "ensure-version/smart-update2-11189/smart-update", "cluster": "smart-update", "namespace": "smart-update2-11189"}
2026-04-09T04:37:49.941Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "ba3000f3-e51b-4ddd-b5b2-ffc5ae9471cb", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-04-09T04:37:49.960Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "ba3000f3-e51b-4ddd-b5b2-ffc5ae9471cb", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-04-09T04:37:50.525Z	ERROR	Reconciler error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "ba3000f3-e51b-4ddd-b5b2-ffc5ae9471cb", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:92\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:25\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:27\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:50\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:543\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:556\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2026-04-09T04:37:50.664Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "4e97f83a-db50-44e7-a015-373c0387d9c9", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-04-09T04:37:50.792Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "4e97f83a-db50-44e7-a015-373c0387d9c9", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-04-09T04:37:50.960Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "4e97f83a-db50-44e7-a015-373c0387d9c9", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T04:37:51.137Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "4e97f83a-db50-44e7-a015-373c0387d9c9", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T04:37:51.211Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "4e97f83a-db50-44e7-a015-373c0387d9c9", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T04:37:51.340Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "4e97f83a-db50-44e7-a015-373c0387d9c9", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T04:37:52.350Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "b2cc2bca-8c58-4ee0-89db-2ca2846bc6a0", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-04-09T04:37:52.367Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "b2cc2bca-8c58-4ee0-89db-2ca2846bc6a0", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-04-09T04:39:16.057Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "b2c90584-2b37-4692-8d89-705fac589a3f", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 34.118.235.12:3306: connect: connection refused"}
2026-04-09T04:41:57.707Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "4a67dd07-a9a3-4892-a77d-28b229bbcb45", "new version": "8.0.41-32.1"}
2026-04-09T04:43:23.131Z	INFO	add new job	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "bad08f02-1f9d-4472-a73b-ef24a182f083", "name": "ensure-version/smart-update2-11189/smart-update", "schedule": "* * * * *"}
2026-04-09T04:44:23.000Z	DEBUG	Use version service endpoint	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "bad08f02-1f9d-4472-a73b-ef24a182f083", "endpoint": "http://version-service.smart-update2-11189.svc.cluster.local:11000"}
2026-04-09T04:44:23.042Z	ERROR	failed to ensure version	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "bad08f02-1f9d-4472-a73b-ef24a182f083", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=4ff72b91-18b3-4fe7-b53d-3fc83b643c5b&databaseVersion=8.0.41-32.1&kubeVersion=v1.32.13-gke.1205000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=4ff72b91-18b3-4fe7-b53d-3fc83b643c5b&databaseVersion=8.0.41-32.1&kubeVersion=v1.32.13-gke.1205000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:300\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:302\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2026-04-09T04:45:23.001Z	INFO	cluster is not ready	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "bad08f02-1f9d-4472-a73b-ef24a182f083"}
2026-04-09T04:45:34.910Z	ERROR	Update status	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "0f50d585-c696-4b80-907f-93ac3b96531f", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"}
2026-04-09T04:45:49.663Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "388708a9-0f6f-44a4-ab22-b9d850efe12a", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-04-09T04:45:49.683Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "388708a9-0f6f-44a4-ab22-b9d850efe12a", "object": "auto-smart-update-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-04-09T04:45:50.254Z	ERROR	Reconciler error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "388708a9-0f6f-44a4-ab22-b9d850efe12a", "error": "failed to deploy haproxy: updatePod for haproxy: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-smart-update-pxc\" already exists", "errorVerbose": "configmaps \"auto-smart-update-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:92\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:25\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:27\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:48\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:50\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:542\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nupdatePod for haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:543\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:555\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to deploy haproxy\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:556\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:367\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2026-04-09T04:45:50.370Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "657dc7d5-e761-49ac-8cbe-ba0cfe8cb483", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-04-09T04:45:50.405Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "657dc7d5-e761-49ac-8cbe-ba0cfe8cb483", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-04-09T04:45:50.460Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "657dc7d5-e761-49ac-8cbe-ba0cfe8cb483", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T04:45:50.519Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "657dc7d5-e761-49ac-8cbe-ba0cfe8cb483", "object": "smart-update-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T04:45:50.578Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "657dc7d5-e761-49ac-8cbe-ba0cfe8cb483", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T04:45:50.661Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "657dc7d5-e761-49ac-8cbe-ba0cfe8cb483", "object": "smart-update-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T04:45:51.491Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "1d5ca30f-6b0b-4064-a929-fff4bbbef6c6", "object": "smart-update-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-04-09T04:45:51.510Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "1d5ca30f-6b0b-4064-a929-fff4bbbef6c6", "object": "smart-update-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-04-09T04:49:44.626Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "67f724b4-83e4-4d0f-80eb-8573b615a359", "new version": "8.0.41-32.1"}
2026-04-09T04:51:19.801Z	INFO	add new job	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "07dbf5c3-9082-47c2-bf02-51ca7d77252b", "name": "ensure-version/smart-update2-11189/smart-update", "schedule": "* * * * *"}
2026-04-09T04:52:19.000Z	DEBUG	Use version service endpoint	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "07dbf5c3-9082-47c2-bf02-51ca7d77252b", "endpoint": "http://version-service.smart-update2-11189.svc.cluster.local:11000"}
2026-04-09T04:52:19.051Z	ERROR	failed to ensure version	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "07dbf5c3-9082-47c2-bf02-51ca7d77252b", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.41-32.1?clusterWideEnabled=true&customResourceUid=44b8e239-3cd6-4531-a84d-754d3693cb15&databaseVersion=8.0.41-32.1&kubeVersion=v1.32.13-gke.1205000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.41-32.1?clusterWideEnabled=true&customResourceUid=44b8e239-3cd6-4531-a84d-754d3693cb15&databaseVersion=8.0.41-32.1&kubeVersion=v1.32.13-gke.1205000&platform=kubernetes&userManagementEnabled=false\": dial tcp: lookup version-service on 34.118.224.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:300\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:302\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2026-04-09T04:53:26.268Z	ERROR	Update status	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "aeb9ea09-032f-4fdd-9cfd-d7604baa607c", "error": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found"}
2026-04-09T04:53:26.268Z	INFO	cluster is not found, deleting the job	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"smart-update","namespace":"smart-update2-11189"}, "namespace": "smart-update2-11189", "name": "smart-update", "reconcileID": "07dbf5c3-9082-47c2-bf02-51ca7d77252b", "name": "ensure-version/smart-update2-11189/smart-update", "cluster": "smart-update", "namespace": "smart-update2-11189"}
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1
github.com/robfig/cron/v3.(*Cron).startJob.func1
github.com/robfig/cron/v3.FuncJob.Run
	/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136
	/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:495
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:259
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:484
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:214
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
+ kubectl get pxc --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
No resources found
+ kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.qsljsC9OT0
++ mktemp
+ local LAST_ERR=/tmp/tmp.4hg7cE0Wch
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.qsljsC9OT0
No resources found
+ cat /tmp/tmp.4hg7cE0Wch
+ rm /tmp/tmp.qsljsC9OT0 /tmp/tmp.4hg7cE0Wch
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.tFRuuLMSNM
++ mktemp
+ local LAST_ERR=/tmp/tmp.9Q1aBZtfe6
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.tFRuuLMSNM
No resources found
+ cat /tmp/tmp.9Q1aBZtfe6
+ rm /tmp/tmp.tFRuuLMSNM /tmp/tmp.9Q1aBZtfe6
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.X9EcxDriOS
++ mktemp
+ local LAST_ERR=/tmp/tmp.v7dLWcvdLZ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.X9EcxDriOS
No resources found
+ cat /tmp/tmp.v7dLWcvdLZ
+ rm /tmp/tmp.X9EcxDriOS /tmp/tmp.v7dLWcvdLZ
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.SYEAAQXek6
++ mktemp
+ local LAST_ERR=/tmp/tmp.d0mBotGQtY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.SYEAAQXek6
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.d0mBotGQtY
+ rm /tmp/tmp.SYEAAQXek6 /tmp/tmp.d0mBotGQtY
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml
namespace "cert-manager" deleted
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace smart-update2-11189
+ rm -rf /tmp/tmp.oGaB4XvPf0
++ mktemp
+ local LAST_OUT=/tmp/tmp.7lw17iBBjI
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

++ mktemp
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
+ local LAST_ERR=/tmp/tmp.qN08JcHuIs
+ local exit_status=0
+ local LAST_OUT=/tmp/tmp.QPrag9jBZS
++ seq 0 2
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace smart-update2-11189
+ local LAST_ERR=/tmp/tmp.0gRPKs7fi5
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator