Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/logs/smart-update2-8-0.log
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
+ API=pxc.percona.com/v9-9-9
+ TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ CLUSTER=smart-update
+ CLUSTER_SIZE=3
+ PROXY_SIZE=2
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]]
++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/'
++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ PXC_VER=8.0
+ TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ VS_URL=http://version-service
+ VS_PORT=11000
+ VS_ENDPOINT=http://version-service:11000
+ main
+ create_infra smart-update2-21243
+ local ns=smart-update2-21243
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.M5a0gh3Z7Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.261mK9GgPv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.M5a0gh3Z7Q
No resources found
+ cat /tmp/tmp.261mK9GgPv
+ rm /tmp/tmp.M5a0gh3Z7Q /tmp/tmp.261mK9GgPv
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.L3tNWBLaAZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.KE0qQY3qX7
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.L3tNWBLaAZ
No resources found
+ cat /tmp/tmp.KE0qQY3qX7
+ rm /tmp/tmp.L3tNWBLaAZ /tmp/tmp.KE0qQY3qX7
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.9F0mqdqtSK
++ mktemp
+ local LAST_ERR=/tmp/tmp.2LgIYqtbvV
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.9F0mqdqtSK
No resources found
+ cat /tmp/tmp.2LgIYqtbvV
+ rm /tmp/tmp.9F0mqdqtSK /tmp/tmp.2LgIYqtbvV
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl api-resources
++ grep chaos-mesh
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrolebinding
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ awk '{print$1}'
++ mktemp
+ xargs kubectl delete ns
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ local LAST_OUT=/tmp/tmp.dTSfTTUpzD
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.9jVFtNfMaP
+ local exit_status=0
++ seq 0 2
+ local LAST_OUT=/tmp/tmp.zMlK8tnbS6
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
++ mktemp
+ local LAST_ERR=/tmp/tmp.mj822x9KqS
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.zMlK8tnbS6
+ cat /tmp/tmp.mj822x9KqS
+ rm /tmp/tmp.zMlK8tnbS6 /tmp/tmp.mj822x9KqS
+ return 0
namespace "cert-manager" deleted
namespace "smart-update2-32102" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.dTSfTTUpzD
namespace "pxc-operator" deleted
+ cat /tmp/tmp.9jVFtNfMaP
+ rm /tmp/tmp.dTSfTTUpzD /tmp/tmp.9jVFtNfMaP
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'namespace/pxc-operator - '
namespace/pxc-operator - + set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.XNd9p1sEIp
++ mktemp
+ local LAST_ERR=/tmp/tmp.l5MJF03BDX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XNd9p1sEIp
namespace/pxc-operator created
+ cat /tmp/tmp.l5MJF03BDX
+ rm /tmp/tmp.XNd9p1sEIp /tmp/tmp.l5MJF03BDX
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oistDqfzc2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1iZGAkMoYc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.oistDqfzc2
++ cat /tmp/tmp.1iZGAkMoYc
++ rm /tmp/tmp.oistDqfzc2 /tmp/tmp.1iZGAkMoYc
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster2 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.uuwJLIp7qZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.kjYO33VzYi
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster2 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.uuwJLIp7qZ
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster2" modified.
+ cat /tmp/tmp.kjYO33VzYi
+ rm /tmp/tmp.uuwJLIp7qZ /tmp/tmp.kjYO33VzYi
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.8yZuxnTaI2
++ mktemp
+ local LAST_ERR=/tmp/tmp.vj95nGCBz3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.8yZuxnTaI2
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.vj95nGCBz3
+ rm /tmp/tmp.8yZuxnTaI2 /tmp/tmp.vj95nGCBz3
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ kubectl_bin apply -f -
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/cw-rbac.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.il1dBuxTez
++ mktemp
+ local LAST_ERR=/tmp/tmp.pitGcKRigC
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.il1dBuxTez
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.pitGcKRigC
+ rm /tmp/tmp.il1dBuxTez /tmp/tmp.pitGcKRigC
+ return 0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/cw-operator.yaml
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.xKfSYt0HLb
++ mktemp
+ local LAST_ERR=/tmp/tmp.VClJZyQOmK
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xKfSYt0HLb
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.VClJZyQOmK
+ rm /tmp/tmp.xKfSYt0HLb /tmp/tmp.VClJZyQOmK
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.FBozhrNwOy
++ mktemp
+ local LAST_ERR=/tmp/tmp.ELvDup7Lw1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.FBozhrNwOy
pod/percona-xtradb-cluster-operator-675fc7d9f5-wmsxj condition met
+ cat /tmp/tmp.ELvDup7Lw1
+ rm /tmp/tmp.FBozhrNwOy /tmp/tmp.ELvDup7Lw1
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xVtgNGHybi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8aA3xlbBNk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xVtgNGHybi
++ cat /tmp/tmp.8aA3xlbBNk
++ rm /tmp/tmp.xVtgNGHybi /tmp/tmp.8aA3xlbBNk
++ return 0
+ wait_pod percona-xtradb-cluster-operator-675fc7d9f5-wmsxj 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-675fc7d9f5-wmsxj
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-675fc7d9f5-wmsxj
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-675fc7d9f5-wmsxj condition met
percona-xtradb-cluster-operator-675fc7d9f5-wmsxj.Ok
+ sleep 3
+ create_namespace smart-update2-21243
+ local namespace=smart-update2-21243
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ sed s/NAMESPACE//
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrolebinding
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrole
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces smart-update2-21243'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces smart-update2-21243
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace smart-update2-21243
+ xargs kubectl delete ns
+ awk '{print$1}'
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.vfAOLldbIR
++ mktemp
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ local LAST_ERR=/tmp/tmp.R2nfIZjAWm
+ local exit_status=0
+ local LAST_OUT=/tmp/tmp.XVKZprDbCK
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.w6XxQxSOfB
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace smart-update2-21243
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vfAOLldbIR
+ cat /tmp/tmp.R2nfIZjAWm
+ rm /tmp/tmp.vfAOLldbIR /tmp/tmp.R2nfIZjAWm
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace smart-update2-21243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace smart-update2-21243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.XVKZprDbCK
+ cat /tmp/tmp.w6XxQxSOfB
Error from server (NotFound): namespaces "smart-update2-21243" not found
+ rm /tmp/tmp.XVKZprDbCK /tmp/tmp.w6XxQxSOfB
+ return 1
+ :
+ wait_for_delete namespace/smart-update2-21243
+ local res=namespace/smart-update2-21243
+ echo -n 'namespace/smart-update2-21243 - '
namespace/smart-update2-21243 - + set +o xtrace
Error from server (NotFound): namespaces "smart-update2-21243" not found
+ desc 'create namespace smart-update2-21243'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace smart-update2-21243
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace smart-update2-21243
++ mktemp
+ local LAST_OUT=/tmp/tmp.A9VKifPIzT
++ mktemp
+ local LAST_ERR=/tmp/tmp.5CgL6rVDe4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace smart-update2-21243
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.A9VKifPIzT
namespace/smart-update2-21243 created
+ cat /tmp/tmp.5CgL6rVDe4
+ rm /tmp/tmp.A9VKifPIzT /tmp/tmp.5CgL6rVDe4
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZcP8QE4w3N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pVx61ACHNX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZcP8QE4w3N
++ cat /tmp/tmp.pVx61ACHNX
++ rm /tmp/tmp.ZcP8QE4w3N /tmp/tmp.pVx61ACHNX
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster2 --namespace=smart-update2-21243
++ mktemp
+ local LAST_OUT=/tmp/tmp.AeS0bp7PW7
++ mktemp
+ local LAST_ERR=/tmp/tmp.qVzgI0zu6F
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster2 --namespace=smart-update2-21243
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.AeS0bp7PW7
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster2" modified.
+ cat /tmp/tmp.qVzgI0zu6F
+ rm /tmp/tmp.AeS0bp7PW7 /tmp/tmp.qVzgI0zu6F
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.7KNYyE7Nu8
++ mktemp
+ local LAST_ERR=/tmp/tmp.hJu4pUokcE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7KNYyE7Nu8
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.hJu4pUokcE
+ rm /tmp/tmp.7KNYyE7Nu8 /tmp/tmp.hJu4pUokcE
+ return 0
+ deploy_version_service
+ desc 'install version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install version service
-----------------------------------------------------------------------------------

+ kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json
++ mktemp
+ local LAST_OUT=/tmp/tmp.j0dZoDV26Y
++ mktemp
+ local LAST_ERR=/tmp/tmp.d2uRwxZp65
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.j0dZoDV26Y
configmap/versions created
+ cat /tmp/tmp.d2uRwxZp65
+ rm /tmp/tmp.j0dZoDV26Y /tmp/tmp.d2uRwxZp65
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/conf/vs.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.vpDouWpMVg
++ mktemp
+ local LAST_ERR=/tmp/tmp.2fQmxYXl3p
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/conf/vs.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vpDouWpMVg
deployment.apps/version-service created
service/version-service created
+ cat /tmp/tmp.2fQmxYXl3p
+ rm /tmp/tmp.vpDouWpMVg /tmp/tmp.2fQmxYXl3p
+ return 0
+ sleep 10
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.l3ttNc0tyV
++ mktemp
+ local LAST_ERR=/tmp/tmp.eAFtsJADai
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.l3ttNc0tyV
namespace/cert-manager created
+ cat /tmp/tmp.eAFtsJADai
+ rm /tmp/tmp.l3ttNc0tyV /tmp/tmp.eAFtsJADai
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.gxox5H2PAK
++ mktemp
+ local LAST_ERR=/tmp/tmp.RB1qSZdcPr
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gxox5H2PAK
namespace/cert-manager labeled
+ cat /tmp/tmp.RB1qSZdcPr
+ rm /tmp/tmp.gxox5H2PAK /tmp/tmp.RB1qSZdcPr
+ return 0
+ kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.Dp91mW55Sg
++ mktemp
+ local LAST_ERR=/tmp/tmp.i9uBmLrFRx
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Dp91mW55Sg
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews configured
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection configured
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.i9uBmLrFRx
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.Dp91mW55Sg /tmp/tmp.i9uBmLrFRx
+ return 0
+ '[' '' == 4.10 ']'
+ sleep 70
++ sort -V
++ grep :8.0
++ head -n1
++ jq -r '.versions[].matrix.pxc[].imagePath'
++ tail -n3
+++ get_operator_pod
+++ local label_prefix=app.kubernetes.io/
++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++++ grep -c percona-xtradb-cluster-operator
+++ local check_label=1
+++ [[ 1 -eq 0 ]]
+++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.TitODFe075
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NEUDY5bmy5
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.TitODFe075
+++ cat /tmp/tmp.NEUDY5bmy5
+++ rm /tmp/tmp.TitODFe075 /tmp/tmp.NEUDY5bmy5
+++ return 0
++ kubectl_bin exec -ti percona-xtradb-cluster-operator-675fc7d9f5-wmsxj -n pxc-operator -- curl -s http://version-service.smart-update2-21243.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VO7HHDQ7Ip
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VstE8BOp1l
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -ti percona-xtradb-cluster-operator-675fc7d9f5-wmsxj -n pxc-operator -- curl -s http://version-service.smart-update2-21243.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VO7HHDQ7Ip
++ cat /tmp/tmp.VstE8BOp1l
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.VO7HHDQ7Ip /tmp/tmp.VstE8BOp1l
++ return 0
+ IMAGE_PXC=percona/percona-xtradb-cluster:8.0.19-10.1
+ kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Kfiwj3akJc
++ mktemp
+ local LAST_ERR=/tmp/tmp.oW6jWUyFj5
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Kfiwj3akJc
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched
+ cat /tmp/tmp.oW6jWUyFj5
+ rm /tmp/tmp.Kfiwj3akJc /tmp/tmp.oW6jWUyFj5
+ return 0
+ kubectl_bin -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-21243.svc.cluster.local:11000
++ mktemp
+ local LAST_OUT=/tmp/tmp.F3C6hML5IU
++ mktemp
+ local LAST_ERR=/tmp/tmp.2Ui4lF946y
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-21243.svc.cluster.local:11000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.F3C6hML5IU
deployment.apps/percona-xtradb-cluster-operator env updated
+ cat /tmp/tmp.2Ui4lF946y
+ rm /tmp/tmp.F3C6hML5IU /tmp/tmp.2Ui4lF946y
+ return 0
+ desc 'Starting telemetry testing'
+ set +o xtrace


-----------------------------------------------------------------------------------
Starting telemetry testing
-----------------------------------------------------------------------------------

+ /usr/bin/sed s/version-service/version-service-cr/g /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/conf/vs.yml
+ kubectl_bin apply -f -
++ mktemp
++ yq 'select(.kind == "Deployment").spec.template.spec.containers[0].image' /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/conf/vs.yml
+ local LAST_OUT=/tmp/tmp.qwL4rtSh1e
++ mktemp
+ local LAST_ERR=/tmp/tmp.UAUHr4leVH
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ yq eval '(. | select(.kind == "Deployment") | .spec.template.spec.containers[0].image) = "perconalab/version-service:main-e378a19"'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.qwL4rtSh1e
deployment.apps/version-service-cr created
service/version-service-cr created
+ cat /tmp/tmp.UAUHr4leVH
+ rm /tmp/tmp.qwL4rtSh1e /tmp/tmp.UAUHr4leVH
+ return 0
+ kubectl_bin delete pod -l run=version-service
++ mktemp
+ local LAST_OUT=/tmp/tmp.RDVjtT3iuO
++ mktemp
+ local LAST_ERR=/tmp/tmp.2vzp5JWbSx
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -l run=version-service
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.RDVjtT3iuO
pod "version-service-6d49c56954-gtq5h" deleted
+ cat /tmp/tmp.2vzp5JWbSx
+ rm /tmp/tmp.RDVjtT3iuO /tmp/tmp.2vzp5JWbSx
+ return 0
++ echo -n percona/percona-xtradb-cluster:8.0.19-10.1
++ sed -r 's/^.*:([0-9]+.[0-9]+).*/\1/'
+ IMAGE_PREFIX=8.0
+ desc 'Enable telemetry on operator level'
+ set +o xtrace


-----------------------------------------------------------------------------------
Enable telemetry on operator level
-----------------------------------------------------------------------------------

+ yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "false"'
+ kubectl_bin apply -n pxc-operator -f -
+ kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.LhfWFmpQIB
++ mktemp
+ local LAST_OUT=/tmp/tmp.7zjb7MgIsf
++ mktemp
+ local LAST_ERR=/tmp/tmp.vOaOWk6FeY
+ local exit_status=0
++ mktemp
+ local LAST_ERR=/tmp/tmp.VzOqAFZ4JB
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n pxc-operator -f -
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.LhfWFmpQIB
+ cat /tmp/tmp.vOaOWk6FeY
+ rm /tmp/tmp.LhfWFmpQIB /tmp/tmp.vOaOWk6FeY
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7zjb7MgIsf
deployment.apps/percona-xtradb-cluster-operator configured
+ cat /tmp/tmp.VzOqAFZ4JB
+ rm /tmp/tmp.7zjb7MgIsf /tmp/tmp.VzOqAFZ4JB
+ return 0
+ sleep 30
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6l475Qi1wi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o0OL6GjBA4
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6l475Qi1wi
++ cat /tmp/tmp.o0OL6GjBA4
++ rm /tmp/tmp.6l475Qi1wi /tmp/tmp.o0OL6GjBA4
++ return 0
+ wait_pod percona-xtradb-cluster-operator-84667df54b-lgzks 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-84667df54b-lgzks
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-84667df54b-lgzks
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-84667df54b-lgzks condition met
percona-xtradb-cluster-operator-84667df54b-lgzks.Ok
+ check_telemetry_transfer http://version-service-cr.smart-update2-21243.svc.cluster.local:11000 disabled enabled
+ local cr_vs_uri=http://version-service-cr.smart-update2-21243.svc.cluster.local:11000
+ local cr_vs_channel=disabled
+ local telemetry_state=enabled
+ desc 'create PXC minimal cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC minimal cluster
-----------------------------------------------------------------------------------

+ cluster=minimal-cluster
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.gcmux4QczI
++ mktemp
+ local LAST_ERR=/tmp/tmp.r7R0Wc0Baa
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gcmux4QczI
deployment.apps/pxc-client created
+ cat /tmp/tmp.r7R0Wc0Baa
+ rm /tmp/tmp.gcmux4QczI /tmp/tmp.r7R0Wc0Baa
+ return 0
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml
+ yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.5UwGemI0n5
++ mktemp
+ local LAST_ERR=/tmp/tmp.vHnOtnAcSH
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5UwGemI0n5
secret/minimal-cluster created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.vHnOtnAcSH
+ rm /tmp/tmp.5UwGemI0n5 /tmp/tmp.vHnOtnAcSH
+ return 0
+ yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-21243.svc.cluster.local:11000"'
+ yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517"'
+ yq eval '.spec.logcollector.image="perconalab/percona-xtradb-cluster-operator:main-logcollector"'
+ kubectl_bin apply -f -
+ yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"'
+ yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.19-10.1"'
+ yq eval '.spec.crVersion="9.9.9"'
+ yq eval '.spec.upgradeOptions.apply="disabled"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.QsbauF2BDe
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/cr-minimal.yaml
++ mktemp
+ local LAST_ERR=/tmp/tmp.3t2WxeaWNe
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QsbauF2BDe
perconaxtradbcluster.pxc.percona.com/minimal-cluster created
+ cat /tmp/tmp.3t2WxeaWNe
+ rm /tmp/tmp.QsbauF2BDe /tmp/tmp.3t2WxeaWNe
+ return 0
+ desc 'check if Pod is started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod is started
-----------------------------------------------------------------------------------

+ wait_for_running minimal-cluster-pxc 1
+ local name=minimal-cluster-pxc
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod minimal-cluster-pxc-0 480
+ local pod=minimal-cluster-pxc-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
Error from server (NotFound): pods "minimal-cluster-pxc-0" not found
minimal-cluster-pxc-0..........................................Ok
+ sleep 20
+ local proxy
++ get_proxy minimal-cluster
++ local target_cluster=minimal-cluster
+++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.NSOJm7mLPA
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.RkjtSqHtmp
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.NSOJm7mLPA
+++ cat /tmp/tmp.RkjtSqHtmp
+++ rm /tmp/tmp.NSOJm7mLPA /tmp/tmp.RkjtSqHtmp
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo minimal-cluster-haproxy
++ return
+ proxy=minimal-cluster-haproxy
+ wait_for_running minimal-cluster-haproxy 1
+ local name=minimal-cluster-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod minimal-cluster-haproxy-0 480
+ local pod=minimal-cluster-haproxy-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/minimal-cluster-haproxy-0 condition met
minimal-cluster-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init)
.Ok
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3OUvhxkJYr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tTL41QycRv
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3OUvhxkJYr
++ cat /tmp/tmp.tTL41QycRv
++ rm /tmp/tmp.3OUvhxkJYr /tmp/tmp.tTL41QycRv
++ return 0
+ client_pod=pxc-client-fdcf9cfbb-4ks57
+ wait_pod pxc-client-fdcf9cfbb-4ks57
+ local pod=pxc-client-fdcf9cfbb-4ks57
+ local max_retry=480
+ local ns=
++ echo pxc-client-fdcf9cfbb-4ks57
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-fdcf9cfbb-4ks57 condition met
pxc-client-fdcf9cfbb-4ks57.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument
command terminated with exit code 5
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DCcVLkQF7D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zaikSESBNL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DCcVLkQF7D
++ cat /tmp/tmp.zaikSESBNL
++ rm /tmp/tmp.DCcVLkQF7D /tmp/tmp.zaikSESBNL
++ return 0
+ client_pod=pxc-client-fdcf9cfbb-4ks57
+ wait_pod pxc-client-fdcf9cfbb-4ks57
+ local pod=pxc-client-fdcf9cfbb-4ks57
+ local max_retry=480
+ local ns=
++ echo pxc-client-fdcf9cfbb-4ks57
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-fdcf9cfbb-4ks57 condition met
pxc-client-fdcf9cfbb-4ks57.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument
command terminated with exit code 5
+ grep -E 'server request payload|unary call'
+ grep -Eo '\{.*\}'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.time_ms")'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ kubectl_bin logs version-service-cr-77d9b658bb-px2cn
++ mktemp
+ local LAST_OUT=/tmp/tmp.XCm2VaAol4
++ mktemp
+ local LAST_ERR=/tmp/tmp.6CIyEZ80s3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs version-service-cr-77d9b658bb-px2cn
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XCm2VaAol4
+ cat /tmp/tmp.6CIyEZ80s3
+ rm /tmp/tmp.XCm2VaAol4 /tmp/tmp.6CIyEZ80s3
+ return 0
+ grep -Eo '\{.*\}'
+ grep -E 'server request payload|unary call'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.time_ms")'
+ kubectl_bin logs version-service-6d49c56954-nshgt
++ mktemp
+ local LAST_OUT=/tmp/tmp.cu6rT2YNYR
++ mktemp
+ local LAST_ERR=/tmp/tmp.by9kBPEHl6
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs version-service-6d49c56954-nshgt
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.cu6rT2YNYR
+ cat /tmp/tmp.by9kBPEHl6
+ rm /tmp/tmp.cu6rT2YNYR /tmp/tmp.by9kBPEHl6
+ return 0
+ local telemetry_log_file=enabled_telemetry.version-service-cw.log.json
+ desc 'telemetry was disabled in CR but in operator not'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR but in operator not
-----------------------------------------------------------------------------------

+ '[' disabled == disabled -a enabled == enabled ']'
+ desc 'operator fallback VS should have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
operator fallback VS should have telemetry
-----------------------------------------------------------------------------------

+ diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json /dev/fd/63
++ grep -f /tmp/tmp.SEJrP012A0/enabled_telemetry.version-service.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json
+ desc 'CR VS should not have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
CR VS should not have telemetry
-----------------------------------------------------------------------------------

+ [[ -s /tmp/tmp.SEJrP012A0/enabled_telemetry.version-service-cr.log.json ]]
+ local image_prefix=disabled
+ local telemetry_cr_log_file=enabled_telemetry.version-service-cr-disabled-cw.log.json
+ desc 'telemetry was disabled in operator but not in CR'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in operator but not in CR
-----------------------------------------------------------------------------------

+ '[' disabled == disabled-recommended -a enabled == disabled ']'
+ desc 'telemetry was disabled in CR as well as in operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR as well as in operator
-----------------------------------------------------------------------------------

+ '[' disabled == disabled -a enabled == disabled ']'
+ kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.P0nk9zLeQy
++ mktemp
+ local LAST_ERR=/tmp/tmp.RSuNuiz9aO
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.P0nk9zLeQy
perconaxtradbcluster.pxc.percona.com/minimal-cluster patched
+ cat /tmp/tmp.RSuNuiz9aO
+ rm /tmp/tmp.P0nk9zLeQy /tmp/tmp.RSuNuiz9aO
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FPjUwBHeH1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XxbA4EceKY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FPjUwBHeH1
++ cat /tmp/tmp.XxbA4EceKY
++ rm /tmp/tmp.FPjUwBHeH1 /tmp/tmp.XxbA4EceKY
++ return 0
+ kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-84667df54b-lgzks
++ mktemp
+ local LAST_OUT=/tmp/tmp.A1GaoEqr92
++ mktemp
+ local LAST_ERR=/tmp/tmp.DmEm6T2sYI
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-84667df54b-lgzks
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.A1GaoEqr92
pod "percona-xtradb-cluster-operator-84667df54b-lgzks" deleted
+ cat /tmp/tmp.DmEm6T2sYI
+ rm /tmp/tmp.A1GaoEqr92 /tmp/tmp.DmEm6T2sYI
+ return 0
+ kubectl_bin delete pxc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.vhQPTWusDG
++ mktemp
+ local LAST_ERR=/tmp/tmp.YKSNcm1rjJ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vhQPTWusDG
perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted
+ cat /tmp/tmp.YKSNcm1rjJ
+ rm /tmp/tmp.vhQPTWusDG /tmp/tmp.YKSNcm1rjJ
+ return 0
+ kubectl_bin delete deploy pxc-client
++ mktemp
+ local LAST_OUT=/tmp/tmp.4L06WLTmwe
++ mktemp
+ local LAST_ERR=/tmp/tmp.tgAFv4pMBC
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete deploy pxc-client
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.4L06WLTmwe
deployment.apps "pxc-client" deleted
+ cat /tmp/tmp.tgAFv4pMBC
+ rm /tmp/tmp.4L06WLTmwe /tmp/tmp.tgAFv4pMBC
+ return 0
+ sleep 30
+ desc 'Disabling telemetry on the operator level'
+ set +o xtrace


-----------------------------------------------------------------------------------
Disabling telemetry on the operator level
-----------------------------------------------------------------------------------

+ kubectl_bin delete pod -l run=version-service-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.Rx4nUb79uo
++ mktemp
+ local LAST_ERR=/tmp/tmp.WKmW1WAeNi
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -l run=version-service-cr
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Rx4nUb79uo
pod "version-service-cr-77d9b658bb-px2cn" deleted
+ cat /tmp/tmp.WKmW1WAeNi
+ rm /tmp/tmp.Rx4nUb79uo /tmp/tmp.WKmW1WAeNi
+ return 0
+ kubectl_bin delete pod -l run=version-service
++ mktemp
+ local LAST_OUT=/tmp/tmp.Qo6JpaFwD1
++ mktemp
+ local LAST_ERR=/tmp/tmp.6ADtJqZ90V
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -l run=version-service
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Qo6JpaFwD1
pod "version-service-6d49c56954-nshgt" deleted
+ cat /tmp/tmp.6ADtJqZ90V
+ rm /tmp/tmp.Qo6JpaFwD1 /tmp/tmp.6ADtJqZ90V
+ return 0
+ kubectl_bin apply -n pxc-operator -f -
+ kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator
+ yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.uX26GoBepZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.z5u0SO4FEl
+ local exit_status=0
++ mktemp
+ local LAST_OUT=/tmp/tmp.hbKXMigUqU
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator
++ mktemp
+ local LAST_ERR=/tmp/tmp.1LpmPrcEJy
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n pxc-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.uX26GoBepZ
+ cat /tmp/tmp.z5u0SO4FEl
+ rm /tmp/tmp.uX26GoBepZ /tmp/tmp.z5u0SO4FEl
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.hbKXMigUqU
deployment.apps/percona-xtradb-cluster-operator configured
+ cat /tmp/tmp.1LpmPrcEJy
+ rm /tmp/tmp.hbKXMigUqU /tmp/tmp.1LpmPrcEJy
+ return 0
+ sleep 30
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iaNE0Zml9i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jFoxZfk9SE
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.iaNE0Zml9i
++ cat /tmp/tmp.jFoxZfk9SE
++ rm /tmp/tmp.iaNE0Zml9i /tmp/tmp.jFoxZfk9SE
++ return 0
+ wait_pod percona-xtradb-cluster-operator-575bd49b49-bnp5m 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-575bd49b49-bnp5m
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-575bd49b49-bnp5m
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-575bd49b49-bnp5m condition met
percona-xtradb-cluster-operator-575bd49b49-bnp5m.Ok
+ check_telemetry_transfer http://version-service-cr.smart-update2-21243.svc.cluster.local:11000 8.0-recommended disabled
+ local cr_vs_uri=http://version-service-cr.smart-update2-21243.svc.cluster.local:11000
+ local cr_vs_channel=8.0-recommended
+ local telemetry_state=disabled
+ desc 'create PXC minimal cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC minimal cluster
-----------------------------------------------------------------------------------

+ cluster=minimal-cluster
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.5GQe11Lg7c
++ mktemp
+ local LAST_ERR=/tmp/tmp.LzPI6sErw4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5GQe11Lg7c
deployment.apps/pxc-client created
+ cat /tmp/tmp.LzPI6sErw4
+ rm /tmp/tmp.5GQe11Lg7c /tmp/tmp.LzPI6sErw4
+ return 0
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml
+ yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.S8oXC7sVIO
++ mktemp
+ local LAST_ERR=/tmp/tmp.6JSmoxmQ07
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.S8oXC7sVIO
secret/minimal-cluster unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.6JSmoxmQ07
+ rm /tmp/tmp.S8oXC7sVIO /tmp/tmp.6JSmoxmQ07
+ return 0
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/cr-minimal.yaml
+ yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-21243.svc.cluster.local:11000"'
+ yq eval '.spec.upgradeOptions.apply="8.0-recommended"'
+ yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.19-10.1"'
+ yq eval '.spec.crVersion="9.9.9"'
+ yq eval '.spec.logcollector.image="perconalab/percona-xtradb-cluster-operator:main-logcollector"'
+ kubectl_bin apply -f -
+ yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"'
+ yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.CLdPD88ErR
++ mktemp
+ local LAST_ERR=/tmp/tmp.OefSjJag3R
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.CLdPD88ErR
perconaxtradbcluster.pxc.percona.com/minimal-cluster created
+ cat /tmp/tmp.OefSjJag3R
+ rm /tmp/tmp.CLdPD88ErR /tmp/tmp.OefSjJag3R
+ return 0
+ desc 'check if Pod is started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod is started
-----------------------------------------------------------------------------------

+ wait_for_running minimal-cluster-pxc 1
+ local name=minimal-cluster-pxc
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod minimal-cluster-pxc-0 480
+ local pod=minimal-cluster-pxc-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-pxc-0
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/minimal-cluster-pxc-0 condition met
minimal-cluster-pxc-0.Ok
+ sleep 20
+ local proxy
++ get_proxy minimal-cluster
++ local target_cluster=minimal-cluster
+++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zYCWVYfutT
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.WZP5LELtqj
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.zYCWVYfutT
+++ cat /tmp/tmp.WZP5LELtqj
+++ rm /tmp/tmp.zYCWVYfutT /tmp/tmp.WZP5LELtqj
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo minimal-cluster-haproxy
++ return
+ proxy=minimal-cluster-haproxy
+ wait_for_running minimal-cluster-haproxy 1
+ local name=minimal-cluster-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod minimal-cluster-haproxy-0 480
+ local pod=minimal-cluster-haproxy-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/minimal-cluster-haproxy-0 condition met
minimal-cluster-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init)
.Ok
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NKUckS2rB2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.S8OoXuPw6j
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NKUckS2rB2
++ cat /tmp/tmp.S8OoXuPw6j
++ rm /tmp/tmp.NKUckS2rB2 /tmp/tmp.S8OoXuPw6j
++ return 0
+ client_pod=pxc-client-fdcf9cfbb-94zc7
+ wait_pod pxc-client-fdcf9cfbb-94zc7
+ local pod=pxc-client-fdcf9cfbb-94zc7
+ local max_retry=480
+ local ns=
++ echo pxc-client-fdcf9cfbb-94zc7
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-fdcf9cfbb-94zc7 condition met
pxc-client-fdcf9cfbb-94zc7.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument
command terminated with exit code 5
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oUrpqrEpvj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ch30iD6PbM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.oUrpqrEpvj
++ cat /tmp/tmp.ch30iD6PbM
++ rm /tmp/tmp.oUrpqrEpvj /tmp/tmp.ch30iD6PbM
++ return 0
+ client_pod=pxc-client-fdcf9cfbb-94zc7
+ wait_pod pxc-client-fdcf9cfbb-94zc7
+ local pod=pxc-client-fdcf9cfbb-94zc7
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
++ echo pxc-client-fdcf9cfbb-94zc7
+ local container=
+ set +o xtrace
pod/pxc-client-fdcf9cfbb-94zc7 condition met
pxc-client-fdcf9cfbb-94zc7.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument
command terminated with exit code 5
+ grep -E 'server request payload|unary call'
+ grep -Eo '\{.*\}'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.time_ms")'
++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}'
+ kubectl_bin logs version-service-cr-77d9b658bb-xbnkm
++ mktemp
+ local LAST_OUT=/tmp/tmp.kCj5FhyPGN
++ mktemp
+ local LAST_ERR=/tmp/tmp.7JuuquKr4x
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs version-service-cr-77d9b658bb-xbnkm
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.kCj5FhyPGN
+ cat /tmp/tmp.7JuuquKr4x
+ rm /tmp/tmp.kCj5FhyPGN /tmp/tmp.7JuuquKr4x
+ return 0
+ grep -Eo '\{.*\}'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ jq 'del(."grpc.time_ms")'
+ jq 'del(."grpc.start_time")'
+ grep -E 'server request payload|unary call'
++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ kubectl_bin logs version-service-6d49c56954-65d5t
++ mktemp
+ local LAST_OUT=/tmp/tmp.vpXP6NQDHd
++ mktemp
+ local LAST_ERR=/tmp/tmp.eSdgK4HFEA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs version-service-6d49c56954-65d5t
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vpXP6NQDHd
+ cat /tmp/tmp.eSdgK4HFEA
+ rm /tmp/tmp.vpXP6NQDHd /tmp/tmp.eSdgK4HFEA
+ return 0
+ local telemetry_log_file=disabled_telemetry.version-service-cw.log.json
+ desc 'telemetry was disabled in CR but in operator not'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR but in operator not
-----------------------------------------------------------------------------------

+ '[' 8.0-recommended == disabled -a disabled == enabled ']'
+ local image_prefix=8.0
+ local telemetry_cr_log_file=disabled_telemetry.version-service-cr-8.0-cw.log.json
+ desc 'telemetry was disabled in operator but not in CR'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in operator but not in CR
-----------------------------------------------------------------------------------

+ '[' 8.0-recommended == 8.0-recommended -a disabled == disabled ']'
+ desc 'cr VS should have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
cr VS should have telemetry
-----------------------------------------------------------------------------------

+ diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.0-cw.log.json /dev/fd/63
++ grep -f /tmp/tmp.SEJrP012A0/disabled_telemetry.version-service-cr.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.0-cw.log.json
+ desc 'operator VS should not have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
operator VS should not have telemetry
-----------------------------------------------------------------------------------

+ [[ -s /tmp/tmp.SEJrP012A0/disabled_telemetry.version-service.log.json ]]
+ desc 'telemetry was disabled in CR as well as in operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR as well as in operator
-----------------------------------------------------------------------------------

+ '[' 8.0-recommended == disabled -a disabled == disabled ']'
+ kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.EWNXOfNmXK
++ mktemp
+ local LAST_ERR=/tmp/tmp.Pj5vTeQryy
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EWNXOfNmXK
perconaxtradbcluster.pxc.percona.com/minimal-cluster patched
+ cat /tmp/tmp.Pj5vTeQryy
+ rm /tmp/tmp.EWNXOfNmXK /tmp/tmp.Pj5vTeQryy
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G0XpPwfUgd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xpl6DJDGjy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.G0XpPwfUgd
++ cat /tmp/tmp.xpl6DJDGjy
++ rm /tmp/tmp.G0XpPwfUgd /tmp/tmp.xpl6DJDGjy
++ return 0
+ kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-575bd49b49-bnp5m
++ mktemp
+ local LAST_OUT=/tmp/tmp.vM1spFeVyo
++ mktemp
+ local LAST_ERR=/tmp/tmp.TLd4hxHyJH
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-575bd49b49-bnp5m
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vM1spFeVyo
pod "percona-xtradb-cluster-operator-575bd49b49-bnp5m" deleted
+ cat /tmp/tmp.TLd4hxHyJH
+ rm /tmp/tmp.vM1spFeVyo /tmp/tmp.TLd4hxHyJH
+ return 0
+ kubectl_bin delete pxc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.w0pIIydYiK
++ mktemp
+ local LAST_ERR=/tmp/tmp.ebJeCQNq6D
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.w0pIIydYiK
perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted
+ cat /tmp/tmp.ebJeCQNq6D
+ rm /tmp/tmp.w0pIIydYiK /tmp/tmp.ebJeCQNq6D
+ return 0
+ kubectl_bin delete deploy pxc-client
++ mktemp
+ local LAST_OUT=/tmp/tmp.bVpDMGLJxg
++ mktemp
+ local LAST_ERR=/tmp/tmp.nr0O0lT9Rx
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete deploy pxc-client
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.bVpDMGLJxg
deployment.apps "pxc-client" deleted
+ cat /tmp/tmp.nr0O0lT9Rx
+ rm /tmp/tmp.bVpDMGLJxg /tmp/tmp.nr0O0lT9Rx
+ return 0
+ sleep 30
+ kubectl_bin delete pod -l run=version-service-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.MSNLImnNoP
++ mktemp
+ local LAST_ERR=/tmp/tmp.JoOAFsVtEP
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -l run=version-service-cr
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MSNLImnNoP
pod "version-service-cr-77d9b658bb-xbnkm" deleted
+ cat /tmp/tmp.JoOAFsVtEP
+ rm /tmp/tmp.MSNLImnNoP /tmp/tmp.JoOAFsVtEP
+ return 0
+ kubectl_bin delete pod -l run=version-service
++ mktemp
+ local LAST_OUT=/tmp/tmp.KBkkECTej1
++ mktemp
+ local LAST_ERR=/tmp/tmp.o85NWLw71U
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -l run=version-service
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.KBkkECTej1
pod "version-service-6d49c56954-65d5t" deleted
+ cat /tmp/tmp.o85NWLw71U
+ rm /tmp/tmp.KBkkECTej1 /tmp/tmp.o85NWLw71U
+ return 0
+ check_telemetry_transfer http://version-service-cr.smart-update2-21243.svc.cluster.local:11000 disabled disabled
+ local cr_vs_uri=http://version-service-cr.smart-update2-21243.svc.cluster.local:11000
+ local cr_vs_channel=disabled
+ local telemetry_state=disabled
+ desc 'create PXC minimal cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC minimal cluster
-----------------------------------------------------------------------------------

+ cluster=minimal-cluster
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.RppUZK4WWu
++ mktemp
+ local LAST_ERR=/tmp/tmp.2pHTZasfSz
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.RppUZK4WWu
deployment.apps/pxc-client created
+ cat /tmp/tmp.2pHTZasfSz
+ rm /tmp/tmp.RppUZK4WWu /tmp/tmp.2pHTZasfSz
+ return 0
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml
+ kubectl_bin apply -f -
+ yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.VKsukeJNPg
++ mktemp
+ local LAST_ERR=/tmp/tmp.H3M7nVRopG
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.VKsukeJNPg
secret/minimal-cluster unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.H3M7nVRopG
+ rm /tmp/tmp.VKsukeJNPg /tmp/tmp.H3M7nVRopG
+ return 0
+ yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-21243.svc.cluster.local:11000"'
+ yq eval '.spec.upgradeOptions.apply="disabled"'
+ yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/cr-minimal.yaml
+ kubectl_bin apply -f -
+ yq eval '.spec.crVersion="9.9.9"'
+ yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517"'
+ yq eval '.spec.logcollector.image="perconalab/percona-xtradb-cluster-operator:main-logcollector"'
+ yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.19-10.1"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.KZtrVWow76
+ yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.zJrm832U6D
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.KZtrVWow76
perconaxtradbcluster.pxc.percona.com/minimal-cluster created
+ cat /tmp/tmp.zJrm832U6D
+ rm /tmp/tmp.KZtrVWow76 /tmp/tmp.zJrm832U6D
+ return 0
+ desc 'check if Pod is started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod is started
-----------------------------------------------------------------------------------

+ wait_for_running minimal-cluster-pxc 1
+ local name=minimal-cluster-pxc
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod minimal-cluster-pxc-0 480
+ local pod=minimal-cluster-pxc-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/minimal-cluster-pxc-0 condition met
minimal-cluster-pxc-0.Ok
+ sleep 20
+ local proxy
++ get_proxy minimal-cluster
++ local target_cluster=minimal-cluster
+++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.QxtuhJDgkO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KndU6QVwWU
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.QxtuhJDgkO
+++ cat /tmp/tmp.KndU6QVwWU
+++ rm /tmp/tmp.QxtuhJDgkO /tmp/tmp.KndU6QVwWU
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo minimal-cluster-haproxy
++ return
+ proxy=minimal-cluster-haproxy
+ wait_for_running minimal-cluster-haproxy 1
+ local name=minimal-cluster-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod minimal-cluster-haproxy-0 480
+ local pod=minimal-cluster-haproxy-0
+ local max_retry=480
+ local ns=
++ echo minimal-cluster-haproxy-0
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/minimal-cluster-haproxy-0 condition met
minimal-cluster-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init)
.Ok
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TOu7AGpJif
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ajZogKjc1s
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TOu7AGpJif
++ cat /tmp/tmp.ajZogKjc1s
++ rm /tmp/tmp.TOu7AGpJif /tmp/tmp.ajZogKjc1s
++ return 0
+ client_pod=pxc-client-fdcf9cfbb-5bz6t
+ wait_pod pxc-client-fdcf9cfbb-5bz6t
+ local pod=pxc-client-fdcf9cfbb-5bz6t
+ local max_retry=480
+ local ns=
++ echo pxc-client-fdcf9cfbb-5bz6t
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-fdcf9cfbb-5bz6t condition met
pxc-client-fdcf9cfbb-5bz6t.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument
command terminated with exit code 5
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kc9SjYsgdl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.erkKLJpFzn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.kc9SjYsgdl
++ cat /tmp/tmp.erkKLJpFzn
++ rm /tmp/tmp.kc9SjYsgdl /tmp/tmp.erkKLJpFzn
++ return 0
+ client_pod=pxc-client-fdcf9cfbb-5bz6t
+ wait_pod pxc-client-fdcf9cfbb-5bz6t
+ local pod=pxc-client-fdcf9cfbb-5bz6t
+ local max_retry=480
+ local ns=
++ echo pxc-client-fdcf9cfbb-5bz6t
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-fdcf9cfbb-5bz6t condition met
pxc-client-fdcf9cfbb-5bz6t.Ok
+ set +o xtrace
[ERROR] mysql: option '-P' requires an argument
command terminated with exit code 5
++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ grep -E 'server request payload|unary call'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ jq 'del(."grpc.time_ms")'
+ grep -Eo '\{.*\}'
+ kubectl_bin logs version-service-cr-77d9b658bb-rff59
++ mktemp
+ local LAST_OUT=/tmp/tmp.vsN39zICeT
++ mktemp
+ local LAST_ERR=/tmp/tmp.TZUuo7y0LI
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs version-service-cr-77d9b658bb-rff59
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vsN39zICeT
+ cat /tmp/tmp.TZUuo7y0LI
+ rm /tmp/tmp.vsN39zICeT /tmp/tmp.TZUuo7y0LI
+ return 0
+ grep -E 'server request payload|unary call'
+ jq 'del(."grpc.request.content".msg.kubeVersion)'
+ jq 'del(."grpc.request.content".msg.customResourceUid)'
+ jq 'del(."grpc.start_time")'
+ jq 'del(."grpc.time_ms")'
++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}'
+ grep -Eo '\{.*\}'
+ kubectl_bin logs version-service-6d49c56954-4zwn9
++ mktemp
+ local LAST_OUT=/tmp/tmp.kT2qs8cIDG
++ mktemp
+ local LAST_ERR=/tmp/tmp.QUjxTVeKIv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs version-service-6d49c56954-4zwn9
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.kT2qs8cIDG
+ cat /tmp/tmp.QUjxTVeKIv
+ rm /tmp/tmp.kT2qs8cIDG /tmp/tmp.QUjxTVeKIv
+ return 0
+ local telemetry_log_file=disabled_telemetry.version-service-cw.log.json
+ desc 'telemetry was disabled in CR but in operator not'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR but in operator not
-----------------------------------------------------------------------------------

+ '[' disabled == disabled -a disabled == enabled ']'
+ local image_prefix=disabled
+ local telemetry_cr_log_file=disabled_telemetry.version-service-cr-disabled-cw.log.json
+ desc 'telemetry was disabled in operator but not in CR'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in operator but not in CR
-----------------------------------------------------------------------------------

+ '[' disabled == disabled-recommended -a disabled == disabled ']'
+ desc 'telemetry was disabled in CR as well as in operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
telemetry was disabled in CR as well as in operator
-----------------------------------------------------------------------------------

+ '[' disabled == disabled -a disabled == disabled ']'
+ desc 'CR VS should not have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
CR VS should not have telemetry
-----------------------------------------------------------------------------------

+ [[ -s /tmp/tmp.SEJrP012A0/disabled_telemetry.version-service-cr.log.json ]]
+ desc 'operator VS should not have telemetry'
+ set +o xtrace


-----------------------------------------------------------------------------------
operator VS should not have telemetry
-----------------------------------------------------------------------------------

+ [[ -s /tmp/tmp.SEJrP012A0/disabled_telemetry.version-service.log.json ]]
+ kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.PDCNYbCq82
++ mktemp
+ local LAST_ERR=/tmp/tmp.oiUrMYZggy
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.PDCNYbCq82
perconaxtradbcluster.pxc.percona.com/minimal-cluster patched
+ cat /tmp/tmp.oiUrMYZggy
+ rm /tmp/tmp.PDCNYbCq82 /tmp/tmp.oiUrMYZggy
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mlSinvJzKH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RgxgDZuPCn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.mlSinvJzKH
++ cat /tmp/tmp.RgxgDZuPCn
++ rm /tmp/tmp.mlSinvJzKH /tmp/tmp.RgxgDZuPCn
++ return 0
+ kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-575bd49b49-w5rdl
++ mktemp
+ local LAST_OUT=/tmp/tmp.YZ9siQEyiv
++ mktemp
+ local LAST_ERR=/tmp/tmp.7qtmaJGnuh
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-575bd49b49-w5rdl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.YZ9siQEyiv
pod "percona-xtradb-cluster-operator-575bd49b49-w5rdl" deleted
+ cat /tmp/tmp.7qtmaJGnuh
+ rm /tmp/tmp.YZ9siQEyiv /tmp/tmp.7qtmaJGnuh
+ return 0
+ kubectl_bin delete pxc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.JiZousMu3Z
++ mktemp
+ local LAST_ERR=/tmp/tmp.9kD1KISPFg
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.JiZousMu3Z
perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted
+ cat /tmp/tmp.9kD1KISPFg
+ rm /tmp/tmp.JiZousMu3Z /tmp/tmp.9kD1KISPFg
+ return 0
+ kubectl_bin delete deploy pxc-client
++ mktemp
+ local LAST_OUT=/tmp/tmp.BNmX5LRXRt
++ mktemp
+ local LAST_ERR=/tmp/tmp.elN0whQdlu
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete deploy pxc-client
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.BNmX5LRXRt
deployment.apps "pxc-client" deleted
+ cat /tmp/tmp.elN0whQdlu
+ rm /tmp/tmp.BNmX5LRXRt /tmp/tmp.elN0whQdlu
+ return 0
+ sleep 30
+ kubectl_bin delete deployment version-service-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.tFH7eyuRhg
++ mktemp
+ local LAST_ERR=/tmp/tmp.Xint3oloxv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete deployment version-service-cr
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.tFH7eyuRhg
deployment.apps "version-service-cr" deleted
+ cat /tmp/tmp.Xint3oloxv
+ rm /tmp/tmp.tFH7eyuRhg /tmp/tmp.Xint3oloxv
+ return 0
+ desc 'Telemetry testing finished'
+ set +o xtrace


-----------------------------------------------------------------------------------
Telemetry testing finished
-----------------------------------------------------------------------------------

+ desc 'PXC cluster with version service offline'
+ set +o xtrace


-----------------------------------------------------------------------------------
PXC cluster with version service offline
-----------------------------------------------------------------------------------

+ cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/conf/smart-update-version-service-unreachable.yml /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517"' /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ spinup_pxc smart-update /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ local cluster=smart-update
+ local config=/tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.aBAxo1vGkV
++ mktemp
+ local LAST_ERR=/tmp/tmp.B5yg0MPIKz
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.aBAxo1vGkV
secret/my-cluster-secrets created
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.B5yg0MPIKz
+ rm /tmp/tmp.aBAxo1vGkV /tmp/tmp.B5yg0MPIKz
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ local LAST_OUT=/tmp/tmp.kxsZ5mcXIq
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-21243~
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ local LAST_ERR=/tmp/tmp.t7zT6fp6tD
+ local exit_status=0
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.kxsZ5mcXIq
deployment.apps/pxc-client created
+ cat /tmp/tmp.t7zT6fp6tD
+ rm /tmp/tmp.kxsZ5mcXIq /tmp/tmp.t7zT6fp6tD
+ return 0
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ apply_config /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ '[' -z '' ']'
+ cat_config /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-21243~
++ mktemp
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ local LAST_OUT=/tmp/tmp.dkuAfv5waF
++ mktemp
+ cat /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ local LAST_ERR=/tmp/tmp.Zng2hLG1LQ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.dkuAfv5waF
perconaxtradbcluster.pxc.percona.com/smart-update created
+ cat /tmp/tmp.Zng2hLG1LQ
+ rm /tmp/tmp.dkuAfv5waF /tmp/tmp.Zng2hLG1LQ
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5Wkv0AU8ij
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KEhB3obx1K
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.5Wkv0AU8ij
+++ cat /tmp/tmp.KEhB3obx1K
+++ rm /tmp/tmp.5Wkv0AU8ij /tmp/tmp.KEhB3obx1K
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ local proxy=smart-update-haproxy
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-21243
++ mktemp
+ local LAST_OUT=/tmp/tmp.a1svZXrTjj
++ mktemp
+ local LAST_ERR=/tmp/tmp.OkokzVvYJB
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-21243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-21243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-21243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.a1svZXrTjj
+ cat /tmp/tmp.OkokzVvYJB
error: no matching resources found
+ rm /tmp/tmp.a1svZXrTjj /tmp/tmp.OkokzVvYJB
+ return 1
+ true
+ wait_for_running smart-update-haproxy 1
+ local name=smart-update-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-haproxy-0 480
+ local pod=smart-update-haproxy-0
+ local max_retry=480
+ local ns=
++ echo smart-update-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
Error from server (NotFound): pods "smart-update-haproxy-0" not found
smart-update-haproxy-0.......................................Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init)
.Ok
+ wait_for_running smart-update-pxc 3
+ local name=smart-update-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-0 480
+ local pod=smart-update-pxc-0
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-0 condition met
smart-update-pxc-0.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-1 480
+ local pod=smart-update-pxc-1
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-1 condition met
smart-update-pxc-1.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-2 480
+ local pod=smart-update-pxc-2
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-2 condition met
smart-update-pxc-2.Ok
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yv4c4X0iMh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.j6TKuuSh27
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.yv4c4X0iMh
++ cat /tmp/tmp.j6TKuuSh27
++ rm /tmp/tmp.yv4c4X0iMh /tmp/tmp.j6TKuuSh27
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tGgcpgmUz7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6A68aoBMjr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.tGgcpgmUz7
++ cat /tmp/tmp.6A68aoBMjr
++ rm /tmp/tmp.tGgcpgmUz7 /tmp/tmp.6A68aoBMjr
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.x32tX2oDQH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i2D7At8Ov6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.x32tX2oDQH
++ cat /tmp/tmp.i2D7At8Ov6
++ rm /tmp/tmp.x32tX2oDQH /tmp/tmp.i2D7At8Ov6
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hYItX0K3N6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gF3lrGITyT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hYItX0K3N6
++ cat /tmp/tmp.gF3lrGITyT
++ rm /tmp/tmp.hYItX0K3N6 /tmp/tmp.gF3lrGITyT
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zR5WheQBeD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zW8OpOZzzo
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zR5WheQBeD
++ cat /tmp/tmp.zW8OpOZzzo
++ rm /tmp/tmp.zR5WheQBeD /tmp/tmp.zW8OpOZzzo
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
++ is_keyring_plugin_in_use smart-update
++ local cluster=smart-update
++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wXi4BM1BZq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CuMCvtpNyC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wXi4BM1BZq
++ cat /tmp/tmp.CuMCvtpNyC
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.wXi4BM1BZq /tmp/tmp.CuMCvtpNyC
++ return 0
+ '[' '' ']'
+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0bPOl6fRDf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cvYnDx8JNi
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0bPOl6fRDf
++ cat /tmp/tmp.cvYnDx8JNi
++ rm /tmp/tmp.0bPOl6fRDf /tmp/tmp.cvYnDx8JNi
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yHGGPNTaSW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NZNbb9fThn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.yHGGPNTaSW
++ cat /tmp/tmp.NZNbb9fThn
++ rm /tmp/tmp.yHGGPNTaSW /tmp/tmp.NZNbb9fThn
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.XOvr4s8nbo
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.heVvhtEMln
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.XOvr4s8nbo
+++++ cat /tmp/tmp.heVvhtEMln
+++++ rm /tmp/tmp.XOvr4s8nbo /tmp/tmp.heVvhtEMln
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ecbjooSzeS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0GBlsvh1zC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ecbjooSzeS
++ cat /tmp/tmp.0GBlsvh1zC
++ rm /tmp/tmp.ecbjooSzeS /tmp/tmp.0GBlsvh1zC
++ return 0
+ [[ 2 == \2 ]]
++ kubectl_bin get pxc/smart-update -o 'jsonpath={.spec.pxc.image}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.M5SdJOMcDv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.T9uRB6b8kg
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc/smart-update -o 'jsonpath={.spec.pxc.image}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.M5SdJOMcDv
++ cat /tmp/tmp.T9uRB6b8kg
++ rm /tmp/tmp.M5SdJOMcDv /tmp/tmp.T9uRB6b8kg
++ return 0
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 != \p\e\r\c\o\n\a\/\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\:\8\.\0\.\1\9\-\1\0\.\1 ]]
+ desc 'PXC cluster update with recommended image by version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
PXC cluster update with recommended image by version service
-----------------------------------------------------------------------------------

+ vs_image=recommended
++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password'
++ local 'command=SELECT @@hostname hostname;'
++ local 'uri=-h smart-update-haproxy -uroot -proot_password'
+++ get_client_pod
+++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dgZCc9aR89
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.07l45qijpR
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.dgZCc9aR89
+++ cat /tmp/tmp.07l45qijpR
+++ rm /tmp/tmp.dgZCc9aR89 /tmp/tmp.07l45qijpR
+++ return 0
++ client_pod=pxc-client-6644d8898f-57twr
++ wait_pod pxc-client-6644d8898f-57twr
++ local pod=pxc-client-6644d8898f-57twr
++ local max_retry=480
++ local ns=
+++ echo pxc-client-6644d8898f-57twr
+++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+++ egrep '^(pxc|proxysql)$'
++ local container=
++ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
++ set +o xtrace
+ initial_primary=smart-update-pxc-0
+ kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.JzUUbb0RMl
++ mktemp
+ local LAST_ERR=/tmp/tmp.lbGuR2D3QE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.JzUUbb0RMl
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ cat /tmp/tmp.lbGuR2D3QE
+ rm /tmp/tmp.JzUUbb0RMl /tmp/tmp.lbGuR2D3QE
+ return 0
+ sleep 55
+ check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ local cluster=smart-update
+ local initial_primary=smart-update-pxc-0
+ local pxc_size=3
+ local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ set +x
Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mxcWH7gxBA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kISJDTE5Pm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.mxcWH7gxBA
++ cat /tmp/tmp.kISJDTE5Pm
++ rm /tmp/tmp.mxcWH7gxBA /tmp/tmp.kISJDTE5Pm
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TN4Bn0i2Rv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ph8TudNqcf
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TN4Bn0i2Rv
++ cat /tmp/tmp.ph8TudNqcf
++ rm /tmp/tmp.TN4Bn0i2Rv /tmp/tmp.ph8TudNqcf
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.2E9NhMPaFL
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.zuIJz3GeDz
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.2E9NhMPaFL
+++++ cat /tmp/tmp.zuIJz3GeDz
+++++ rm /tmp/tmp.2E9NhMPaFL /tmp/tmp.zuIJz3GeDz
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u7NMJJvFfB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dtETRetUab
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.u7NMJJvFfB
++ cat /tmp/tmp.dtETRetUab
++ rm /tmp/tmp.u7NMJJvFfB /tmp/tmp.dtETRetUab
++ return 0
+ [[ 2 == \2 ]]
++ seq 0 2
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.R2qmcX086a
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TA380tkEWY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.R2qmcX086a
++ cat /tmp/tmp.TA380tkEWY
++ rm /tmp/tmp.R2qmcX086a /tmp/tmp.TA380tkEWY
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ egrep '^(pxc|proxysql)$'
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BnbSrgNqKp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LrvP2JW63y
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BnbSrgNqKp
++ cat /tmp/tmp.LrvP2JW63y
++ rm /tmp/tmp.BnbSrgNqKp /tmp/tmp.LrvP2JW63y
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8ubr5dvmML
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tAGY72Safy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8ubr5dvmML
++ cat /tmp/tmp.tAGY72Safy
++ rm /tmp/tmp.8ubr5dvmML /tmp/tmp.tAGY72Safy
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ kubectl_bin delete -f /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.wzD7ZmtkJS
++ mktemp
+ local LAST_ERR=/tmp/tmp.989nD9Hb3Q
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.wzD7ZmtkJS
perconaxtradbcluster.pxc.percona.com "smart-update" deleted
+ cat /tmp/tmp.989nD9Hb3Q
+ rm /tmp/tmp.wzD7ZmtkJS /tmp/tmp.989nD9Hb3Q
+ return 0
+ kubectl_bin delete pvc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.VqqyHNJ6jv
++ mktemp
+ local LAST_ERR=/tmp/tmp.zT4xjY15GU
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pvc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.VqqyHNJ6jv
persistentvolumeclaim "datadir-smart-update-pxc-0" deleted
persistentvolumeclaim "datadir-smart-update-pxc-1" deleted
persistentvolumeclaim "datadir-smart-update-pxc-2" deleted
+ cat /tmp/tmp.zT4xjY15GU
+ rm /tmp/tmp.VqqyHNJ6jv /tmp/tmp.zT4xjY15GU
+ return 0
+ desc 'PXC cluster update with the latest image by version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
PXC cluster update with the latest image by version service
-----------------------------------------------------------------------------------

+ spinup_pxc smart-update /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ local cluster=smart-update
+ local config=/tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.UnYG1epVNo
++ mktemp
+ local LAST_ERR=/tmp/tmp.7RipH3Se4F
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UnYG1epVNo
secret/my-cluster-secrets unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.7RipH3Se4F
+ rm /tmp/tmp.UnYG1epVNo /tmp/tmp.7RipH3Se4F
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.nmOM8eQ3lt
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-21243~
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
++ mktemp
+ local LAST_ERR=/tmp/tmp.53nwa8V1oR
+ local exit_status=0
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517#'
++ seq 0 2
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nmOM8eQ3lt
deployment.apps/pxc-client unchanged
+ cat /tmp/tmp.53nwa8V1oR
+ rm /tmp/tmp.nmOM8eQ3lt /tmp/tmp.53nwa8V1oR
+ return 0
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ apply_config /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ '[' -z '' ']'
+ cat_config /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ cat /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-21243~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.I7orNbxEjT
++ mktemp
+ local LAST_ERR=/tmp/tmp.wBKZDqWqDz
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.I7orNbxEjT
perconaxtradbcluster.pxc.percona.com/smart-update created
+ cat /tmp/tmp.wBKZDqWqDz
+ rm /tmp/tmp.I7orNbxEjT /tmp/tmp.wBKZDqWqDz
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.EdlQ28Z59F
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1PhyWd0QHq
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.EdlQ28Z59F
+++ cat /tmp/tmp.1PhyWd0QHq
+++ rm /tmp/tmp.EdlQ28Z59F /tmp/tmp.1PhyWd0QHq
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ local proxy=smart-update-haproxy
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-21243
++ mktemp
+ local LAST_OUT=/tmp/tmp.hPnB4P6Da3
++ mktemp
+ local LAST_ERR=/tmp/tmp.0FqgqD3b5Z
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-21243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-21243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-21243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.hPnB4P6Da3
+ cat /tmp/tmp.0FqgqD3b5Z
error: no matching resources found
+ rm /tmp/tmp.hPnB4P6Da3 /tmp/tmp.0FqgqD3b5Z
+ return 1
+ true
+ wait_for_running smart-update-haproxy 1
+ local name=smart-update-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-haproxy-0 480
+ local pod=smart-update-haproxy-0
+ local max_retry=480
+ local ns=
++ echo smart-update-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/smart-update-haproxy-0 condition met
smart-update-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init)
.Ok
+ wait_for_running smart-update-pxc 3
+ local name=smart-update-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-0 480
+ local pod=smart-update-pxc-0
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-0 condition met
smart-update-pxc-0.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-1 480
+ local pod=smart-update-pxc-1
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-1
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-1 condition met
smart-update-pxc-1.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-2 480
+ local pod=smart-update-pxc-2
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-2 condition met
smart-update-pxc-2.Ok
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ifcz6VTcAm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zR96oolw3I
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Ifcz6VTcAm
++ cat /tmp/tmp.zR96oolw3I
++ rm /tmp/tmp.Ifcz6VTcAm /tmp/tmp.zR96oolw3I
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BHpD0Ssbs6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q4jSaMfXW5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BHpD0Ssbs6
++ cat /tmp/tmp.Q4jSaMfXW5
++ rm /tmp/tmp.BHpD0Ssbs6 /tmp/tmp.Q4jSaMfXW5
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7ul3PpVMie
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pNfnQcAdbZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7ul3PpVMie
++ cat /tmp/tmp.pNfnQcAdbZ
++ rm /tmp/tmp.7ul3PpVMie /tmp/tmp.pNfnQcAdbZ
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GslwvnPvHY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JYviiYhIYj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GslwvnPvHY
++ cat /tmp/tmp.JYviiYhIYj
++ rm /tmp/tmp.GslwvnPvHY /tmp/tmp.JYviiYhIYj
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
++ echo pxc-client-6644d8898f-57twr
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BmYM0CFUr3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NJfUkH3yWw
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BmYM0CFUr3
++ cat /tmp/tmp.NJfUkH3yWw
++ rm /tmp/tmp.BmYM0CFUr3 /tmp/tmp.NJfUkH3yWw
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ egrep '^(pxc|proxysql)$'
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
++ is_keyring_plugin_in_use smart-update
++ local cluster=smart-update
++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Osz0Jsffd6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OF97qc8016
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Osz0Jsffd6
++ cat /tmp/tmp.OF97qc8016
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.Osz0Jsffd6 /tmp/tmp.OF97qc8016
++ return 0
+ '[' '' ']'
+ vs_image=latest
++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password'
++ local 'command=SELECT @@hostname hostname;'
++ local 'uri=-h smart-update-haproxy -uroot -proot_password'
+++ get_client_pod
+++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.P7qPGLcS0R
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ckYn96sh97
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.P7qPGLcS0R
+++ cat /tmp/tmp.ckYn96sh97
+++ rm /tmp/tmp.P7qPGLcS0R /tmp/tmp.ckYn96sh97
+++ return 0
++ client_pod=pxc-client-6644d8898f-57twr
++ wait_pod pxc-client-6644d8898f-57twr
++ local pod=pxc-client-6644d8898f-57twr
++ local max_retry=480
++ local ns=
+++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+++ echo pxc-client-6644d8898f-57twr
+++ egrep '^(pxc|proxysql)$'
++ local container=
++ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
++ set +o xtrace
+ initial_primary=smart-update-pxc-0
+ kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.TNQSWed5Er
++ mktemp
+ local LAST_ERR=/tmp/tmp.9USrf7vLEE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.TNQSWed5Er
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ cat /tmp/tmp.9USrf7vLEE
+ rm /tmp/tmp.TNQSWed5Er /tmp/tmp.9USrf7vLEE
+ return 0
+ sleep 55
+ check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ local cluster=smart-update
+ local initial_primary=smart-update-pxc-0
+ local pxc_size=3
+ local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ set +x
Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oSiwPTUTvw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TdOncbc1Vg
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.oSiwPTUTvw
++ cat /tmp/tmp.TdOncbc1Vg
++ rm /tmp/tmp.oSiwPTUTvw /tmp/tmp.TdOncbc1Vg
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GGjvdLc6mn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.a3AUHhdXt8
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GGjvdLc6mn
++ cat /tmp/tmp.a3AUHhdXt8
++ rm /tmp/tmp.GGjvdLc6mn /tmp/tmp.a3AUHhdXt8
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.AJ0sRIvJJ0
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.3upSbd9pTZ
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.AJ0sRIvJJ0
+++++ cat /tmp/tmp.3upSbd9pTZ
+++++ rm /tmp/tmp.AJ0sRIvJJ0 /tmp/tmp.3upSbd9pTZ
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wV7r42CXpp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FSo6fCbdkR
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wV7r42CXpp
++ cat /tmp/tmp.FSo6fCbdkR
++ rm /tmp/tmp.wV7r42CXpp /tmp/tmp.FSo6fCbdkR
++ return 0
+ [[ 2 == \2 ]]
++ seq 0 2
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XZQqJEkQbw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8XoKkLgUBH
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XZQqJEkQbw
++ cat /tmp/tmp.8XoKkLgUBH
++ rm /tmp/tmp.XZQqJEkQbw /tmp/tmp.8XoKkLgUBH
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.krsYDDsPGT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O5T2vjqiYM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.krsYDDsPGT
++ cat /tmp/tmp.O5T2vjqiYM
++ rm /tmp/tmp.krsYDDsPGT /tmp/tmp.O5T2vjqiYM
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
++ echo pxc-client-6644d8898f-57twr
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.87qJVc1diB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZI8i1Q5AB7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.87qJVc1diB
++ cat /tmp/tmp.ZI8i1Q5AB7
++ rm /tmp/tmp.87qJVc1diB /tmp/tmp.ZI8i1Q5AB7
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-6644d8898f-57twr
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ kubectl_bin delete -f /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.MZ3xfpRVhZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.9A1Kgeppl1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MZ3xfpRVhZ
perconaxtradbcluster.pxc.percona.com "smart-update" deleted
+ cat /tmp/tmp.9A1Kgeppl1
+ rm /tmp/tmp.MZ3xfpRVhZ /tmp/tmp.9A1Kgeppl1
+ return 0
+ kubectl_bin delete pvc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.7VPVfEAMgG
++ mktemp
+ local LAST_ERR=/tmp/tmp.FtIyxr6sA3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pvc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7VPVfEAMgG
persistentvolumeclaim "datadir-smart-update-pxc-0" deleted
persistentvolumeclaim "datadir-smart-update-pxc-1" deleted
persistentvolumeclaim "datadir-smart-update-pxc-2" deleted
+ cat /tmp/tmp.FtIyxr6sA3
+ rm /tmp/tmp.7VPVfEAMgG /tmp/tmp.FtIyxr6sA3
+ return 0
+ desc 'PXC cluster update with explicitly specified image inside version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
PXC cluster update with explicitly specified image inside version service
-----------------------------------------------------------------------------------

+ spinup_pxc smart-update /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ local cluster=smart-update
+ local config=/tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.NgUroKtckH
++ mktemp
+ local LAST_ERR=/tmp/tmp.bwU6ysmCxz
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.NgUroKtckH
secret/my-cluster-secrets unchanged
secret/some-name-ssl unchanged
secret/some-name-ssl-internal unchanged
+ cat /tmp/tmp.bwU6ysmCxz
+ rm /tmp/tmp.NgUroKtckH /tmp/tmp.bwU6ysmCxz
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ kubectl_bin apply -f -
++ mktemp
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-21243~
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ local LAST_OUT=/tmp/tmp.fh1DNocnzR
++ mktemp
+ local LAST_ERR=/tmp/tmp.aQ3RtnIKpK
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.fh1DNocnzR
deployment.apps/pxc-client unchanged
+ cat /tmp/tmp.aQ3RtnIKpK
+ rm /tmp/tmp.fh1DNocnzR /tmp/tmp.aQ3RtnIKpK
+ return 0
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ apply_config /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ '[' -z '' ']'
+ cat_config /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ cat /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-21243~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ local LAST_OUT=/tmp/tmp.34o9MvwPqK
++ mktemp
+ local LAST_ERR=/tmp/tmp.c4jH4R6Qee
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.34o9MvwPqK
perconaxtradbcluster.pxc.percona.com/smart-update created
+ cat /tmp/tmp.c4jH4R6Qee
+ rm /tmp/tmp.34o9MvwPqK /tmp/tmp.c4jH4R6Qee
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Fji5oRat1y
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bkR4cCqDx6
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.Fji5oRat1y
+++ cat /tmp/tmp.bkR4cCqDx6
+++ rm /tmp/tmp.Fji5oRat1y /tmp/tmp.bkR4cCqDx6
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-haproxy
++ return
+ local proxy=smart-update-haproxy
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-21243
++ mktemp
+ local LAST_OUT=/tmp/tmp.0KQHwGOHLo
++ mktemp
+ local LAST_ERR=/tmp/tmp.00XOOCGvV4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-21243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-21243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-21243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.0KQHwGOHLo
+ cat /tmp/tmp.00XOOCGvV4
error: no matching resources found
+ rm /tmp/tmp.0KQHwGOHLo /tmp/tmp.00XOOCGvV4
+ return 1
+ true
+ wait_for_running smart-update-haproxy 1
+ local name=smart-update-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-haproxy-0 480
+ local pod=smart-update-haproxy-0
+ local max_retry=480
+ local ns=
++ echo smart-update-haproxy-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/smart-update-haproxy-0 condition met
smart-update-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init)
.Ok
+ wait_for_running smart-update-pxc 3
+ local name=smart-update-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-0 480
+ local pod=smart-update-pxc-0
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo smart-update-pxc-0
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-0 condition met
smart-update-pxc-0.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-1 480
+ local pod=smart-update-pxc-1
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-1 condition met
smart-update-pxc-1.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-2 480
+ local pod=smart-update-pxc-2
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-2 condition met
smart-update-pxc-2.Ok
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QpNHLqEpxN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nhrt6hQZ0S
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.QpNHLqEpxN
++ cat /tmp/tmp.nhrt6hQZ0S
++ rm /tmp/tmp.QpNHLqEpxN /tmp/tmp.nhrt6hQZ0S
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NQekALwMmP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vSAqzyFY2g
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NQekALwMmP
++ cat /tmp/tmp.vSAqzyFY2g
++ rm /tmp/tmp.NQekALwMmP /tmp/tmp.vSAqzyFY2g
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8qe82Dqzhv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xvqGJjgN4f
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8qe82Dqzhv
++ cat /tmp/tmp.xvqGJjgN4f
++ rm /tmp/tmp.8qe82Dqzhv /tmp/tmp.xvqGJjgN4f
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BkQhxsXAK2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ICay8qCVnz
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BkQhxsXAK2
++ cat /tmp/tmp.ICay8qCVnz
++ rm /tmp/tmp.BkQhxsXAK2 /tmp/tmp.ICay8qCVnz
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tUdDrOSEpz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r27Fpe0TTi
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.tUdDrOSEpz
++ cat /tmp/tmp.r27Fpe0TTi
++ rm /tmp/tmp.tUdDrOSEpz /tmp/tmp.r27Fpe0TTi
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
++ is_keyring_plugin_in_use smart-update
++ local cluster=smart-update
++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EO2avxWLiB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wGKBADgnPA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.EO2avxWLiB
++ cat /tmp/tmp.wGKBADgnPA
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.EO2avxWLiB /tmp/tmp.wGKBADgnPA
++ return 0
+ '[' '' ']'
++ grep :8.0
++ tail -n2
++ sort -V
++ jq -r '.versions[].matrix.pxc[].imagePath'
+++ get_operator_pod
+++ local label_prefix=app.kubernetes.io/
++ head -n1
++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++++ grep -c percona-xtradb-cluster-operator
+++ local check_label=1
+++ [[ 1 -eq 0 ]]
+++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.CFCdE8TXsr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.WHMZs6NC5L
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.CFCdE8TXsr
+++ cat /tmp/tmp.WHMZs6NC5L
+++ rm /tmp/tmp.CFCdE8TXsr /tmp/tmp.WHMZs6NC5L
+++ return 0
++ kubectl_bin exec -ti percona-xtradb-cluster-operator-575bd49b49-jfqrd -n pxc-operator -- curl -s http://version-service.smart-update2-21243.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JLrOgmOlrM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pyAaDw4JfZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -ti percona-xtradb-cluster-operator-575bd49b49-jfqrd -n pxc-operator -- curl -s http://version-service.smart-update2-21243.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.JLrOgmOlrM
++ cat /tmp/tmp.pyAaDw4JfZ
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.JLrOgmOlrM /tmp/tmp.pyAaDw4JfZ
++ return 0
+ vs_image=percona/percona-xtradb-cluster:8.0.19-10.1
++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password'
++ local 'command=SELECT @@hostname hostname;'
++ local 'uri=-h smart-update-haproxy -uroot -proot_password'
+++ get_client_pod
+++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.npyKBQbmYc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bh66qZiiTR
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.npyKBQbmYc
+++ cat /tmp/tmp.bh66qZiiTR
+++ rm /tmp/tmp.npyKBQbmYc /tmp/tmp.bh66qZiiTR
+++ return 0
++ client_pod=pxc-client-6644d8898f-57twr
++ wait_pod pxc-client-6644d8898f-57twr
++ local pod=pxc-client-6644d8898f-57twr
++ local max_retry=480
++ local ns=
+++ egrep '^(pxc|proxysql)$'
+++ echo pxc-client-6644d8898f-57twr
+++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ local container=
++ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
++ set +o xtrace
+ initial_primary=smart-update-pxc-0
+ kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.19-10.1","schedule": "* * * * *"}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.9eXNJ0kW2U
++ mktemp
+ local LAST_ERR=/tmp/tmp.NmmaPAI0my
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.19-10.1","schedule": "* * * * *"}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.9eXNJ0kW2U
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ cat /tmp/tmp.NmmaPAI0my
+ rm /tmp/tmp.9eXNJ0kW2U /tmp/tmp.NmmaPAI0my
+ return 0
+ sleep 55
+ check_last_pod_to_update smart-update smart-update-pxc-0 3 percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.19-10.1
+ local cluster=smart-update
+ local initial_primary=smart-update-pxc-0
+ local pxc_size=3
+ local target_image=percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.19-10.1
+ set +x
Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2
+ local cluster_name=smart-update
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Qs7Bg5wv9u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G2XH9vtcLx
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Qs7Bg5wv9u
++ cat /tmp/tmp.G2XH9vtcLx
++ rm /tmp/tmp.Qs7Bg5wv9u /tmp/tmp.G2XH9vtcLx
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.33KNKqtWnz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qdNAJpCJOt
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.33KNKqtWnz
++ cat /tmp/tmp.qdNAJpCJOt
++ rm /tmp/tmp.33KNKqtWnz /tmp/tmp.qdNAJpCJOt
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine smart-update
+++ local cluster_name=smart-update
++++ get_proxy smart-update
++++ local target_cluster=smart-update
+++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.iT4sbr5zjW
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.qZstSXs2Lt
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.iT4sbr5zjW
+++++ cat /tmp/tmp.qZstSXs2Lt
+++++ rm /tmp/tmp.iT4sbr5zjW /tmp/tmp.qZstSXs2Lt
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo smart-update-haproxy
++++ return
+++ local cluster_proxy=smart-update-haproxy
+++ echo haproxy
++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IuBc9m0fLY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B7WtNqi8jq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IuBc9m0fLY
++ cat /tmp/tmp.B7WtNqi8jq
++ rm /tmp/tmp.IuBc9m0fLY /tmp/tmp.B7WtNqi8jq
++ return 0
+ [[ 2 == \2 ]]
++ seq 0 2
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.63tMRUKAvi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.S6Abh4zPQJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.63tMRUKAvi
++ cat /tmp/tmp.S6Abh4zPQJ
++ rm /tmp/tmp.63tMRUKAvi /tmp/tmp.S6Abh4zPQJ
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AwoAsakBw3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HtLn5WDKHd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AwoAsakBw3
++ cat /tmp/tmp.HtLn5WDKHd
++ rm /tmp/tmp.AwoAsakBw3 /tmp/tmp.HtLn5WDKHd
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
++ echo pxc-client-6644d8898f-57twr
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ for i in '$(seq 0 $((CLUSTER_SIZE - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GagGXtM7Yd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.btAya8sYTP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GagGXtM7Yd
++ cat /tmp/tmp.btAya8sYTP
++ rm /tmp/tmp.GagGXtM7Yd /tmp/tmp.btAya8sYTP
++ return 0
+ client_pod=pxc-client-6644d8898f-57twr
+ wait_pod pxc-client-6644d8898f-57twr
+ local pod=pxc-client-6644d8898f-57twr
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-57twr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-57twr condition met
pxc-client-6644d8898f-57twr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.SEJrP012A0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.SEJrP012A0/select-1.sql
+ kubectl_bin delete -f /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Gg1xHUk2Ev
++ mktemp
+ local LAST_ERR=/tmp/tmp.oXcRBXBwMv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /tmp/tmp.SEJrP012A0/smart-update-version-service-unreachable.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Gg1xHUk2Ev
perconaxtradbcluster.pxc.percona.com "smart-update" deleted
+ cat /tmp/tmp.oXcRBXBwMv
+ rm /tmp/tmp.Gg1xHUk2Ev /tmp/tmp.oXcRBXBwMv
+ return 0
+ kubectl_bin delete pvc --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.nKcnRRDcNe
++ mktemp
+ local LAST_ERR=/tmp/tmp.lkVVtbok2G
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pvc --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nKcnRRDcNe
persistentvolumeclaim "datadir-smart-update-pxc-0" deleted
persistentvolumeclaim "datadir-smart-update-pxc-1" deleted
persistentvolumeclaim "datadir-smart-update-pxc-2" deleted
+ cat /tmp/tmp.lkVVtbok2G
+ rm /tmp/tmp.nKcnRRDcNe /tmp/tmp.lkVVtbok2G
+ return 0
+ desc cleanup
+ set +o xtrace


-----------------------------------------------------------------------------------
cleanup
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/conf/vs.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ABg1YIj2sb
++ mktemp
+ local LAST_ERR=/tmp/tmp.aqqyrmN096
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/smart-update2/conf/vs.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ABg1YIj2sb
deployment.apps "version-service" deleted
service "version-service" deleted
+ cat /tmp/tmp.aqqyrmN096
+ rm /tmp/tmp.ABg1YIj2sb /tmp/tmp.aqqyrmN096
+ return 0
+ destroy smart-update2-21243
+ local namespace=smart-update2-21243
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
+ grep -v level=info
+ grep -v 'get backup status: Job.batch'
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ grep -v 'the object has been modified'
+ tee /tmp/tmp.SEJrP012A0/operator.log
+ sort -u
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9Ndp0QYqde
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5x4YHuiMTY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.9Ndp0QYqde
++ cat /tmp/tmp.5x4YHuiMTY
++ rm /tmp/tmp.9Ndp0QYqde /tmp/tmp.5x4YHuiMTY
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-575bd49b49-jfqrd
++ mktemp
+ local LAST_OUT=/tmp/tmp.XeGJrk4mRs
++ mktemp
+ local LAST_ERR=/tmp/tmp.LNTOpPpDnx
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-575bd49b49-jfqrd
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XeGJrk4mRs
+ cat /tmp/tmp.LNTOpPpDnx
+ rm /tmp/tmp.XeGJrk4mRs /tmp/tmp.LNTOpPpDnx
+ return 0
2024-07-10T13:31:50.002Z	INFO	setup	Manager starting up	{"gitCommit": "51af051703ea68e6367821ec3068826842ab5c9d", "gitBranch": "PR-1743-51af0517", "buildTime": "2024-07-10T11:38:48Z", "goVersion": "go1.22.5", "os": "linux", "arch": "amd64"}
2024-07-10T13:31:50.002Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.26.15-gke.1469001"}
2024-07-10T13:31:50.003Z	INFO	setup	Registering Components.
2024-07-10T13:31:53.718Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2024-07-10T13:31:53.721Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2024-07-10T13:31:53.721Z	INFO	controller-runtime.metrics	Starting metrics server
2024-07-10T13:31:53.721Z	INFO	setup	Starting the Cmd.
2024-07-10T13:31:53.800Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate
2024-07-10T13:31:53.800Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2024-07-10T13:31:53.800Z	INFO	controller-runtime.webhook	Starting webhook server
2024-07-10T13:31:53.800Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2024-07-10T13:31:53.801Z	INFO	controller-runtime.certwatcher	Starting certificate watcher
2024-07-10T13:31:53.902Z	INFO	attempting to acquire leader lease pxc-operator/08db1feb.percona.com...
2024-07-10T13:32:09.927Z	DEBUG	events	percona-xtradb-cluster-operator-575bd49b49-jfqrd_7afa5acb-e9e4-48fc-8053-8e33d0154500 became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"a1f91d92-8de2-4874-b45e-a66b32be5c67","apiVersion":"coordination.k8s.io/v1","resourceVersion":"58569"}, "reason": "LeaderElection"}
2024-07-10T13:32:09.927Z	INFO	Starting Controller	{"controller": "pxcbackup-controller"}
2024-07-10T13:32:09.927Z	INFO	Starting Controller	{"controller": "pxc-controller"}
2024-07-10T13:32:09.927Z	INFO	Starting Controller	{"controller": "pxcrestore-controller"}
2024-07-10T13:32:09.927Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2024-07-10T13:32:09.927Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"}
2024-07-10T13:32:09.927Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2024-07-10T13:32:09.927Z	INFO	successfully acquired lease pxc-operator/08db1feb.percona.com
2024-07-10T13:32:10.140Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "worker count": 1}
2024-07-10T13:32:10.140Z	INFO	Starting workers	{"controller": "pxc-controller", "worker count": 1}
2024-07-10T13:32:10.140Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "worker count": 1}
2024-07-10T13:32:10.672Z	ERROR	Update status	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "minimal-cluster", "reconcileID": "37a9b1e3-deac-4fea-93fe-3102fad788ef", "error": "get pxc status: get statefulset: StatefulSet.apps \"minimal-cluster-pxc\" not found", "errorVerbose": "StatefulSet.apps \"minimal-cluster-pxc\" not found\nget statefulset\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).appStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:188\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:102\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:203\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nget pxc status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:104\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:203\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-10T13:32:55.228Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "f9d51e0f-96c3-42de-85ab-cf91630e94d8", "object": "smart-update-pxc"}
2024-07-10T13:32:55.283Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "f9d51e0f-96c3-42de-85ab-cf91630e94d8", "object": "smart-update-pxc"}
2024-07-10T13:32:55.283Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "f9d51e0f-96c3-42de-85ab-cf91630e94d8"}
2024-07-10T13:32:55.366Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "f9d51e0f-96c3-42de-85ab-cf91630e94d8", "object": "smart-update-pxc-unready"}
2024-07-10T13:32:55.428Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "f9d51e0f-96c3-42de-85ab-cf91630e94d8", "object": "smart-update-haproxy"}
2024-07-10T13:32:55.599Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "f9d51e0f-96c3-42de-85ab-cf91630e94d8", "object": "smart-update-haproxy"}
2024-07-10T13:32:55.749Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "f9d51e0f-96c3-42de-85ab-cf91630e94d8", "object": "smart-update-haproxy"}
2024-07-10T13:32:55.821Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "f9d51e0f-96c3-42de-85ab-cf91630e94d8", "object": "smart-update-haproxy-replicas"}
2024-07-10T13:32:55.924Z	INFO	KubeAPIWarningLogger	.metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 22dc3a05-f7d0-4379-9177-1da556a4e4e8
2024-07-10T13:32:56.156Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "a7161131-c587-4a5c-a99d-fa88b8575914", "object": "smart-update-pxc"}
2024-07-10T13:32:56.195Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "a7161131-c587-4a5c-a99d-fa88b8575914", "object": "smart-update-haproxy"}
2024-07-10T13:32:56.210Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "a7161131-c587-4a5c-a99d-fa88b8575914"}
2024-07-10T13:33:01.070Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "773b0d83-9f69-46d7-9dd4-bb0c6ffaf106"}
2024-07-10T13:33:06.132Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "da84af00-52d8-4946-b693-04a6766c47b2"}
2024-07-10T13:33:11.195Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "02ffca5f-9c6e-48dd-b726-1e3af747a768"}
2024-07-10T13:33:16.331Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "a36aea31-2d78-4a3d-9e2b-eb48071bab5e"}
2024-07-10T13:33:21.396Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "633c6a21-9ea7-480b-8de7-5b55ff1a52cb"}
2024-07-10T13:33:26.460Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "c3605c7b-50c1-4714-aaf6-6f05ae80c328"}
2024-07-10T13:33:31.539Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "72b345d8-9012-4f83-a29a-581235b94afa"}
2024-07-10T13:33:36.621Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "0b4bb664-0ae9-4b72-ac6f-a15dbb8d7ae2"}
2024-07-10T13:33:41.689Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "496c636b-1e03-4f41-a087-22cf4a841d58"}
2024-07-10T13:33:46.762Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "25c58b80-fd70-4729-bc8d-68eec407fb69"}
2024-07-10T13:33:51.840Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "019e2496-4d85-43a3-a06c-9d04810d56d4"}
2024-07-10T13:33:56.905Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "658df37a-58c0-4ca1-b9e6-f72ba20a34db"}
2024-07-10T13:34:01.964Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "0910592d-2e5b-4c6a-9bda-965f6cf9854c"}
2024-07-10T13:34:07.020Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "ca50c9b8-7697-4d96-a8d6-d39a8549cf86"}
2024-07-10T13:34:12.101Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "3acb420c-308a-4e44-93a4-ae1b237b7bc5"}
2024-07-10T13:34:12.313Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "dc792f5e-09bc-4c6b-9f90-ff9f66825281", "user": "operator"}
2024-07-10T13:34:12.363Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "dc792f5e-09bc-4c6b-9f90-ff9f66825281", "user": "monitor"}
2024-07-10T13:34:12.416Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "dc792f5e-09bc-4c6b-9f90-ff9f66825281"}
2024-07-10T13:34:12.534Z	INFO	monitor user privileges granted	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "dc792f5e-09bc-4c6b-9f90-ff9f66825281"}
2024-07-10T13:34:12.618Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "dc792f5e-09bc-4c6b-9f90-ff9f66825281", "user": "xtrabackup"}
2024-07-10T13:34:12.695Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "dc792f5e-09bc-4c6b-9f90-ff9f66825281"}
2024-07-10T13:34:12.749Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "dc792f5e-09bc-4c6b-9f90-ff9f66825281", "user": "replication"}
2024-07-10T13:34:12.785Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "dc792f5e-09bc-4c6b-9f90-ff9f66825281"}
2024-07-10T13:34:17.306Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "78a1b86c-51fd-43fb-a54c-bcbcac79875d"}
2024-07-10T13:34:22.475Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "454dce8a-6ee3-4d21-b31f-dd06faa74aec"}
2024-07-10T13:34:27.640Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "f053979e-3371-4441-8937-e630897806bd"}
2024-07-10T13:34:32.779Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "73f0e442-7da4-403c-98df-f147a996d888"}
2024-07-10T13:36:37.480Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "39188cd3-d39a-45f2-ae21-d788110af716", "user": "root"}
2024-07-10T13:36:38.649Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "39188cd3-d39a-45f2-ae21-d788110af716", "new version": "8.0.19-10"}
2024-07-10T13:38:10.321Z	INFO	add new job	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "5a5ff936-1b4d-4dc2-98ca-2b2d9324c37f", "name": "ensure-version/smart-update2-21243/smart-update", "schedule": "* * * * *"}
2024-07-10T13:39:10.000Z	DEBUG	Use version service endpoint	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "5a5ff936-1b4d-4dc2-98ca-2b2d9324c37f", "endpoint": "http://version-service.smart-update2-21243.svc.cluster.local:11000"}
2024-07-10T13:39:10.126Z	ERROR	failed to ensure version	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "5a5ff936-1b4d-4dc2-98ca-2b2d9324c37f", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=22dc3a05-f7d0-4379-9177-1da556a4e4e8&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1469001&platform=kubernetes\": dial tcp: lookup version-service on 10.14.176.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=22dc3a05-f7d0-4379-9177-1da556a4e4e8&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1469001&platform=kubernetes\": dial tcp: lookup version-service on 10.14.176.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:266\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:285\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-10T13:39:37.774Z	INFO	Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "ee4356b4-287c-4fbd-a063-d633e1e24ef8"}
2024-07-10T13:39:37.856Z	INFO	Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "7c543dc1-49ab-4083-bb17-0175bbad7768"}
2024-07-10T13:40:12.474Z	ERROR	Update status	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "387130bc-bafd-45f3-8bd6-787ea4c232ed", "error": "write status: PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found", "errorVerbose": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found\nwrite status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:157\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:140\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:203\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-10T13:40:12.475Z	INFO	cluster is not found, deleting the job	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "5a5ff936-1b4d-4dc2-98ca-2b2d9324c37f", "name": "ensure-version/smart-update2-21243/smart-update", "cluster": "smart-update", "namespace": "smart-update2-21243"}
2024-07-10T13:40:34.653Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "1b41ed49-1545-4dd6-b06f-072ea6ab2c81", "object": "smart-update-pxc"}
2024-07-10T13:40:34.681Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "1b41ed49-1545-4dd6-b06f-072ea6ab2c81", "object": "smart-update-pxc"}
2024-07-10T13:40:34.748Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "1b41ed49-1545-4dd6-b06f-072ea6ab2c81", "object": "smart-update-haproxy"}
2024-07-10T13:40:34.791Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "1b41ed49-1545-4dd6-b06f-072ea6ab2c81"}
2024-07-10T13:40:34.792Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "1b41ed49-1545-4dd6-b06f-072ea6ab2c81", "object": "smart-update-pxc"}
2024-07-10T13:40:34.954Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "1b41ed49-1545-4dd6-b06f-072ea6ab2c81", "object": "smart-update-pxc-unready"}
2024-07-10T13:40:35.115Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "1b41ed49-1545-4dd6-b06f-072ea6ab2c81", "object": "smart-update-haproxy"}
2024-07-10T13:40:35.192Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "1b41ed49-1545-4dd6-b06f-072ea6ab2c81", "object": "smart-update-haproxy"}
2024-07-10T13:40:35.273Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "1b41ed49-1545-4dd6-b06f-072ea6ab2c81", "object": "smart-update-haproxy-replicas"}
2024-07-10T13:40:35.330Z	INFO	KubeAPIWarningLogger	.metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 3830bad2-e586-4cd6-a93f-548ff04c8422
2024-07-10T13:40:35.446Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "e5b33b5f-6a7a-442c-b28e-0f92bee22b46"}
2024-07-10T13:40:37.510Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "22055593-d430-45ff-a3d5-c0d548ade984"}
2024-07-10T13:40:42.648Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "0ee99a6c-f924-4439-b5bb-5d026557698d"}
2024-07-10T13:40:47.729Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "757201aa-8b18-45d5-badc-d9823c40835c"}
2024-07-10T13:40:52.800Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "62d9de3d-0a7b-40f7-90a4-431050d8b067"}
2024-07-10T13:40:57.870Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "34d0fa56-1338-4778-9c4a-b1b5f550226c"}
2024-07-10T13:41:02.937Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "a951dac6-4716-4d0f-b534-e6fa13cc861c"}
2024-07-10T13:41:08.002Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "5763f738-b353-44fe-827c-59916f13ca5a"}
2024-07-10T13:41:13.072Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "bb647daf-bf72-4688-8877-abcfc8e0492e"}
2024-07-10T13:41:18.143Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "b56e1ea8-7d36-4335-a873-5446107e5adb"}
2024-07-10T13:41:23.209Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "59f83682-1934-4ff3-acc0-f7ce7f732eb1"}
2024-07-10T13:41:28.273Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "1eca0247-991a-4dc5-a4d2-c72ec007951d"}
2024-07-10T13:41:33.356Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "b405a492-3648-4d41-92c3-7edf0b515c34"}
2024-07-10T13:41:38.418Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "47b19be3-f4c0-42d2-b5db-beb61760b51d"}
2024-07-10T13:41:43.860Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "6f769acf-5bec-45fb-adbb-95c626e388ce"}
2024-07-10T13:41:49.177Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "1dc9e01c-4b07-459c-b874-ea859fc5b26f"}
2024-07-10T13:41:49.324Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "99100c6a-e48b-4b7e-b306-27cfdcfbf3af"}
2024-07-10T13:41:51.525Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "f9c973fa-0144-4703-93f4-532423144f3e"}
2024-07-10T13:41:54.327Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "f284b802-53ad-46fe-95ed-414f6f3d2491"}
2024-07-10T13:41:59.535Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "2b5b02e1-b9f3-4af6-9aaf-d3c36f945d2f"}
2024-07-10T13:42:04.791Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "972ac6d0-40d1-4cc4-9a94-11dde97572c8"}
2024-07-10T13:42:09.968Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "b9d33d0e-9274-40d9-b108-6d268ca6d7d3"}
2024-07-10T13:42:15.124Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "8b0223ed-a957-4642-9779-241c09929ccf"}
2024-07-10T13:44:21.577Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "9f8c0060-b1b7-462c-b679-90c3786e5d88", "new version": "8.0.19-10"}
2024-07-10T13:45:50.492Z	INFO	add new job	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "7b47e592-553a-4e41-aef1-1c1c66e637b3", "name": "ensure-version/smart-update2-21243/smart-update", "schedule": "* * * * *"}
2024-07-10T13:46:50.001Z	DEBUG	Use version service endpoint	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "7b47e592-553a-4e41-aef1-1c1c66e637b3", "endpoint": "http://version-service.smart-update2-21243.svc.cluster.local:11000"}
2024-07-10T13:46:50.038Z	ERROR	failed to ensure version	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "7b47e592-553a-4e41-aef1-1c1c66e637b3", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=3830bad2-e586-4cd6-a93f-548ff04c8422&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1469001&platform=kubernetes\": dial tcp: lookup version-service on 10.14.176.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=3830bad2-e586-4cd6-a93f-548ff04c8422&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1469001&platform=kubernetes\": dial tcp: lookup version-service on 10.14.176.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:266\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:285\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-10T13:47:50.001Z	INFO	cluster is not ready	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "7b47e592-553a-4e41-aef1-1c1c66e637b3"}
2024-07-10T13:47:57.422Z	ERROR	Update status	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "865f97cd-e1ed-4231-b649-1eb7c2f2d612", "error": "write status: PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found", "errorVerbose": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found\nwrite status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:157\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:140\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:203\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-10T13:48:20.048Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "6f93084d-1de7-48d8-b7cc-ce2744a33811", "object": "smart-update-pxc"}
2024-07-10T13:48:20.070Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "6f93084d-1de7-48d8-b7cc-ce2744a33811", "object": "smart-update-pxc"}
2024-07-10T13:48:20.182Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "6f93084d-1de7-48d8-b7cc-ce2744a33811"}
2024-07-10T13:48:20.183Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "6f93084d-1de7-48d8-b7cc-ce2744a33811", "object": "smart-update-pxc"}
2024-07-10T13:48:20.290Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "6f93084d-1de7-48d8-b7cc-ce2744a33811", "object": "smart-update-pxc-unready"}
2024-07-10T13:48:20.393Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "6f93084d-1de7-48d8-b7cc-ce2744a33811", "object": "smart-update-haproxy"}
2024-07-10T13:48:20.501Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "6f93084d-1de7-48d8-b7cc-ce2744a33811", "object": "smart-update-haproxy"}
2024-07-10T13:48:20.768Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "6f93084d-1de7-48d8-b7cc-ce2744a33811", "object": "smart-update-haproxy-replicas"}
2024-07-10T13:48:20.946Z	INFO	KubeAPIWarningLogger	.metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 741fb21d-0a38-4d58-890a-013170357a4e
2024-07-10T13:48:21.079Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "a6716892-c942-4670-ba72-160221badc1d", "object": "smart-update-haproxy"}
2024-07-10T13:48:21.090Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "a6716892-c942-4670-ba72-160221badc1d"}
2024-07-10T13:48:22.455Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "c8bd4b17-a409-4568-858a-4c7eb11cd04d"}
2024-07-10T13:48:27.516Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "bfc3cb92-6b4f-4ee1-a77c-652526f17f09"}
2024-07-10T13:48:32.576Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "f6c368a1-6340-4ea8-9feb-56a3353f5497"}
2024-07-10T13:48:37.642Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "0da7c943-834b-4e43-ad54-4dccb5597183"}
2024-07-10T13:48:42.699Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "645f2453-93ac-4405-a206-4741ecc2aff6"}
2024-07-10T13:48:47.767Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "2a03704e-5cdc-4a38-937b-d7a3981deae9"}
2024-07-10T13:48:52.828Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "84daeb3b-b672-4eb8-ae85-41fe2c23b70a"}
2024-07-10T13:48:57.924Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "5c5e5347-65e1-4482-950b-e4d11007349f"}
2024-07-10T13:49:02.982Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "ceae0150-f3a8-4c6f-9e97-fd4a9e3c5ea8"}
2024-07-10T13:49:08.051Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "65bd331b-73f2-4d01-9d9c-54bf89e6e2bf"}
2024-07-10T13:49:13.172Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "48775a4f-ba60-4014-92b2-df1d31fdb21a"}
2024-07-10T13:49:18.241Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "9e3c82ec-a792-4429-81b3-df1b540b0b2b"}
2024-07-10T13:49:23.313Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "927129e9-3e95-4ede-9012-9c218f30da94"}
2024-07-10T13:49:28.549Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "23a5269f-53f3-498d-a87d-5f3ee73d0262"}
2024-07-10T13:49:33.679Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "794b92e9-7260-4f77-8cdb-beae3c04a550"}
2024-07-10T13:49:33.866Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "d9e24571-cd07-4491-a757-f8da16c9e7ee"}
2024-07-10T13:49:35.910Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "d9e24571-cd07-4491-a757-f8da16c9e7ee", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.14.180.176:3306: connect: connection refused"}
2024-07-10T13:49:38.838Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "51e4a779-6b2c-4f11-9dd0-0222aa9c48be"}
2024-07-10T13:49:39.020Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "9719e767-925d-4398-91f2-419a35eed1b6"}
2024-07-10T13:49:44.122Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "c9e9243d-e22b-4d4d-a91a-ce23beedef22"}
2024-07-10T13:49:49.303Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "985f66c6-c5ea-4bc5-9049-774a709d44b6"}
2024-07-10T13:49:54.500Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "e569bfb9-2c8a-4b99-8509-fb78a6e09d32"}
2024-07-10T13:49:59.672Z	INFO	Waiting for HAProxy to be ready before smart update	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "50df0e59-c8a8-48bd-b593-24acfe924e1f"}
2024-07-10T13:52:06.860Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "f7cb45f3-16a8-4b71-952d-8d9eaae812a6", "new version": "8.0.19-10"}
2024-07-10T13:53:35.475Z	INFO	add new job	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "98b540ad-0034-459a-b1c2-ef8111c13cfe", "name": "ensure-version/smart-update2-21243/smart-update", "schedule": "* * * * *"}
2024-07-10T13:54:35.000Z	DEBUG	Use version service endpoint	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "98b540ad-0034-459a-b1c2-ef8111c13cfe", "endpoint": "http://version-service.smart-update2-21243.svc.cluster.local:11000"}
2024-07-10T13:54:35.058Z	ERROR	failed to ensure version	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "98b540ad-0034-459a-b1c2-ef8111c13cfe", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.19-10.1?clusterWideEnabled=true&customResourceUid=741fb21d-0a38-4d58-890a-013170357a4e&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1469001&platform=kubernetes\": dial tcp: lookup version-service on 10.14.176.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.19-10.1?clusterWideEnabled=true&customResourceUid=741fb21d-0a38-4d58-890a-013170357a4e&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1469001&platform=kubernetes\": dial tcp: lookup version-service on 10.14.176.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:266\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:285\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-10T13:55:35.562Z	ERROR	Update status	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "cdb886e3-d173-4851-8475-72fdfc4d62b5", "error": "write status: PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found", "errorVerbose": "PerconaXtraDBCluster.pxc.percona.com \"smart-update\" not found\nwrite status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:157\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:140\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:203\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-10T13:55:35.563Z	INFO	cluster is not found, deleting the job	{"controller": "pxc-controller", "namespace": "smart-update2-21243", "name": "smart-update", "reconcileID": "98b540ad-0034-459a-b1c2-ef8111c13cfe", "name": "ensure-version/smart-update2-21243/smart-update", "cluster": "smart-update", "namespace": "smart-update2-21243"}
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1
github.com/robfig/cron/v3.(*Cron).startJob.func1
github.com/robfig/cron/v3.FuncJob.Run
	/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136
	/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:205
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:265
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:214
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.bOrTtU1uho
++ mktemp
+ local LAST_ERR=/tmp/tmp.s6XkbSCmV7
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.bOrTtU1uho
No resources found
+ cat /tmp/tmp.s6XkbSCmV7
+ rm /tmp/tmp.bOrTtU1uho /tmp/tmp.s6XkbSCmV7
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.2v4Qj58qzX
++ mktemp
+ local LAST_ERR=/tmp/tmp.yauGNgV39z
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2v4Qj58qzX
No resources found
+ cat /tmp/tmp.yauGNgV39z
+ rm /tmp/tmp.2v4Qj58qzX /tmp/tmp.yauGNgV39z
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.saQAE0cqUq
++ mktemp
+ local LAST_ERR=/tmp/tmp.epk2ysqsKp
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.saQAE0cqUq
No resources found
+ cat /tmp/tmp.epk2ysqsKp
+ rm /tmp/tmp.saQAE0cqUq /tmp/tmp.epk2ysqsKp
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.u0PnhrKPeN
++ mktemp
+ local LAST_ERR=/tmp/tmp.PyE0wj7Rcx
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.u0PnhrKPeN
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.PyE0wj7Rcx
+ rm /tmp/tmp.u0PnhrKPeN /tmp/tmp.PyE0wj7Rcx
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace smart-update2-21243
+ rm -rf /tmp/tmp.SEJrP012A0
++ mktemp
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.avVRq9pUnp
+ local LAST_OUT=/tmp/tmp.3RUSX1R7kl
+ desc 'test passed'
++ mktemp
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

++ mktemp
+ local LAST_ERR=/tmp/tmp.fzD6IFgOdZ
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.nHfohOGLgX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace smart-update2-21243
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator