Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/logs/smart-update1-8-0.log
WARNING: version difference between client (1.29) and server (1.25) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.29) and server (1.25) exceeds the supported minor version skew of +/-1
+ API=pxc.percona.com/v9-9-9
+ TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ CLUSTER=smart-update
+ CLUSTER_SIZE=3
+ PROXY_SIZE=2
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]]
++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/'
++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ PXC_VER=8.0
+ TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ VS_URL=http://version-service
+ VS_PORT=11000
+ VS_ENDPOINT=http://version-service:11000
+ main
+ create_infra smart-update1-22666
+ local ns=smart-update1-22666
+ '[' -n pxc-operator ']'
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ kubectl patch pxc -n smart-update1-5685 smart-update --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.VcOPaud350
++ mktemp
+ local LAST_ERR=/tmp/tmp.0Z7d0YWBlF
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.VcOPaud350
perconaxtradbcluster.pxc.percona.com "smart-update" deleted
+ cat /tmp/tmp.0Z7d0YWBlF
+ rm /tmp/tmp.VcOPaud350 /tmp/tmp.0Z7d0YWBlF
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.V08q3Hf8hG
++ mktemp
+ local LAST_ERR=/tmp/tmp.FKzEXgi8dH
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.V08q3Hf8hG
No resources found
+ cat /tmp/tmp.FKzEXgi8dH
+ rm /tmp/tmp.V08q3Hf8hG /tmp/tmp.FKzEXgi8dH
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.moJ1QC8BOt
++ mktemp
+ local LAST_ERR=/tmp/tmp.3iN3qr0XQF
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.moJ1QC8BOt
No resources found
+ cat /tmp/tmp.3iN3qr0XQF
+ rm /tmp/tmp.moJ1QC8BOt /tmp/tmp.3iN3qr0XQF
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
+ '[' -n '' ']'
++ awk '{print $1}'
++ grep chaos-mesh.org
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get MutatingWebhookConfiguration
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ xargs kubectl delete ns
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.SKCwS4r0cE
++ mktemp
+ local LAST_ERR=/tmp/tmp.wT0LRolD3E
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ awk '{print$1}'
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.CBhcoHdoG8
++ mktemp
+ local LAST_ERR=/tmp/tmp.vYzupPx722
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.SKCwS4r0cE
+ cat /tmp/tmp.wT0LRolD3E
+ rm /tmp/tmp.SKCwS4r0cE /tmp/tmp.wT0LRolD3E
+ return 0
namespace "cert-manager" deleted
namespace "smart-update1-5685" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.CBhcoHdoG8
namespace "pxc-operator" deleted
+ cat /tmp/tmp.vYzupPx722
+ rm /tmp/tmp.CBhcoHdoG8 /tmp/tmp.vYzupPx722
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'namespace/pxc-operator - '
namespace/pxc-operator - + set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.XxPbwtwgJ5
++ mktemp
+ local LAST_ERR=/tmp/tmp.3aI8KpReNr
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XxPbwtwgJ5
namespace/pxc-operator created
+ cat /tmp/tmp.3aI8KpReNr
+ rm /tmp/tmp.XxPbwtwgJ5 /tmp/tmp.3aI8KpReNr
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wvrROhioJb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rpHjOPng7d
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wvrROhioJb
++ cat /tmp/tmp.rpHjOPng7d
++ rm /tmp/tmp.wvrROhioJb /tmp/tmp.rpHjOPng7d
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster4 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ETkEgFFqH4
++ mktemp
+ local LAST_ERR=/tmp/tmp.5c2BX6hwDX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster4 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ETkEgFFqH4
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster4" modified.
+ cat /tmp/tmp.5c2BX6hwDX
+ rm /tmp/tmp.ETkEgFFqH4 /tmp/tmp.5c2BX6hwDX
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.BT1Cwmrhrj
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dfnqm6mSFm
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.BT1Cwmrhrj
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.Dfnqm6mSFm
+ rm /tmp/tmp.BT1Cwmrhrj /tmp/tmp.Dfnqm6mSFm
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/deploy/cw-rbac.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.UH0SKVmUVy
++ mktemp
+ local LAST_ERR=/tmp/tmp.aaDiAnTnEL
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UH0SKVmUVy
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.aaDiAnTnEL
+ rm /tmp/tmp.UH0SKVmUVy /tmp/tmp.aaDiAnTnEL
+ return 0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1522-1ed4605b^'
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/deploy/cw-operator.yaml
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' -
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.3IZziHuFHU
++ mktemp
+ local LAST_ERR=/tmp/tmp.rXKkJSj559
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.3IZziHuFHU
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.rXKkJSj559
+ rm /tmp/tmp.3IZziHuFHU /tmp/tmp.rXKkJSj559
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.Frv2lkFBon
++ mktemp
+ local LAST_ERR=/tmp/tmp.nunG1Xjxuh
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Frv2lkFBon
pod/percona-xtradb-cluster-operator-6dbd475d79-shg67 condition met
+ cat /tmp/tmp.nunG1Xjxuh
+ rm /tmp/tmp.Frv2lkFBon /tmp/tmp.nunG1Xjxuh
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.w0TcQSPfwC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sq9kidvhdd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.w0TcQSPfwC
++ cat /tmp/tmp.sq9kidvhdd
++ rm /tmp/tmp.w0TcQSPfwC /tmp/tmp.sq9kidvhdd
++ return 0
+ wait_pod percona-xtradb-cluster-operator-6dbd475d79-shg67 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-6dbd475d79-shg67
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-6dbd475d79-shg67
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-6dbd475d79-shg67 condition met
percona-xtradb-cluster-operator-6dbd475d79-shg67.Ok
+ sleep 3
+ create_namespace smart-update1-22666
+ local namespace=smart-update1-22666
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ tail -n1
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
+ '[' -n '' ']'
++ grep chaos-mesh.org
++ awk '{print $1}'
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces smart-update1-22666'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces smart-update1-22666
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace smart-update1-22666
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.OZVFAKr8Ql
++ mktemp
+ local LAST_ERR=/tmp/tmp.TSqavMXX1D
+ local exit_status=0
++ seq 0 2
+ local LAST_OUT=/tmp/tmp.IuUjOmN4AZ
+ xargs kubectl delete ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace smart-update1-22666
++ mktemp
+ local LAST_ERR=/tmp/tmp.lgmlXzEVl6
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace smart-update1-22666
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.IuUjOmN4AZ
+ cat /tmp/tmp.lgmlXzEVl6
+ rm /tmp/tmp.IuUjOmN4AZ /tmp/tmp.lgmlXzEVl6
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace smart-update1-22666
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.OZVFAKr8Ql
+ cat /tmp/tmp.TSqavMXX1D
Error from server (NotFound): namespaces "smart-update1-22666" not found
+ rm /tmp/tmp.OZVFAKr8Ql /tmp/tmp.TSqavMXX1D
+ return 1
+ :
+ wait_for_delete namespace/smart-update1-22666
+ local res=namespace/smart-update1-22666
+ echo -n 'namespace/smart-update1-22666 - '
namespace/smart-update1-22666 - + set +o xtrace
Error from server (NotFound): namespaces "smart-update1-22666" not found
+ desc 'create namespace smart-update1-22666'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace smart-update1-22666
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace smart-update1-22666
++ mktemp
+ local LAST_OUT=/tmp/tmp.oGCe8YZHjk
++ mktemp
+ local LAST_ERR=/tmp/tmp.6x22SDk6cK
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace smart-update1-22666
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.oGCe8YZHjk
namespace/smart-update1-22666 created
+ cat /tmp/tmp.6x22SDk6cK
+ rm /tmp/tmp.oGCe8YZHjk /tmp/tmp.6x22SDk6cK
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OfNnuDQQCZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yUoQGtCBpy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.OfNnuDQQCZ
++ cat /tmp/tmp.yUoQGtCBpy
++ rm /tmp/tmp.OfNnuDQQCZ /tmp/tmp.yUoQGtCBpy
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster4 --namespace=smart-update1-22666
++ mktemp
+ local LAST_OUT=/tmp/tmp.XydjCJIn7R
++ mktemp
+ local LAST_ERR=/tmp/tmp.ygDN3wnqT3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster4 --namespace=smart-update1-22666
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XydjCJIn7R
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster4" modified.
+ cat /tmp/tmp.ygDN3wnqT3
+ rm /tmp/tmp.XydjCJIn7R /tmp/tmp.ygDN3wnqT3
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.rVHh9Oab4m
++ mktemp
+ local LAST_ERR=/tmp/tmp.rkUlRdly0w
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.rVHh9Oab4m
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.rkUlRdly0w
+ rm /tmp/tmp.rVHh9Oab4m /tmp/tmp.rkUlRdly0w
+ return 0
+ deploy_version_service
+ desc 'install version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install version service
-----------------------------------------------------------------------------------

+ kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json
++ mktemp
+ local LAST_OUT=/tmp/tmp.nARZ54sAmY
++ mktemp
+ local LAST_ERR=/tmp/tmp.K0GUsIULzZ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/conf/operator.9.9.9.pxc-operator.json
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nARZ54sAmY
configmap/versions created
+ cat /tmp/tmp.K0GUsIULzZ
+ rm /tmp/tmp.nARZ54sAmY /tmp/tmp.K0GUsIULzZ
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/conf/vs.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.bVJFeAU76M
++ mktemp
+ local LAST_ERR=/tmp/tmp.pnz880uZV0
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/conf/vs.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.bVJFeAU76M
deployment.apps/version-service created
service/version-service created
+ cat /tmp/tmp.pnz880uZV0
+ rm /tmp/tmp.bVJFeAU76M /tmp/tmp.pnz880uZV0
+ return 0
+ sleep 10
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.krKq5GIHzU
++ mktemp
+ local LAST_ERR=/tmp/tmp.T80KVxoPx1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.krKq5GIHzU
namespace/cert-manager created
+ cat /tmp/tmp.T80KVxoPx1
+ rm /tmp/tmp.krKq5GIHzU /tmp/tmp.T80KVxoPx1
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.gEAFRWkuh9
++ mktemp
+ local LAST_ERR=/tmp/tmp.E0IJ0meSXU
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gEAFRWkuh9
namespace/cert-manager labeled
+ cat /tmp/tmp.E0IJ0meSXU
+ rm /tmp/tmp.gEAFRWkuh9 /tmp/tmp.E0IJ0meSXU
+ return 0
+ kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.12.1/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.v7il3BlNF0
++ mktemp
+ local LAST_ERR=/tmp/tmp.A9dkGGagEf
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.12.1/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.v7il3BlNF0
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
configmap/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews configured
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection configured
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.A9dkGGagEf
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.v7il3BlNF0 /tmp/tmp.A9dkGGagEf
+ return 0
+ '[' '' == 4.10 ']'
+ sleep 70
++ jq -r '.versions[].matrix.pxc[].imagePath'
++ sort -V
+++ get_operator_pod
+++ local label_prefix=app.kubernetes.io/
++ grep :8.0
++ tail -n3
++ head -n1
++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++++ grep -c percona-xtradb-cluster-operator
+++ local check_label=1
+++ [[ 1 -eq 0 ]]
+++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WlkRihCiSB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.H9UTkFHVB8
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.WlkRihCiSB
+++ cat /tmp/tmp.H9UTkFHVB8
+++ rm /tmp/tmp.WlkRihCiSB /tmp/tmp.H9UTkFHVB8
+++ return 0
++ kubectl_bin exec -ti percona-xtradb-cluster-operator-6dbd475d79-shg67 -n pxc-operator -- curl -s http://version-service.smart-update1-22666.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NYsiq1xBS2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mpmV3k3SDC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -ti percona-xtradb-cluster-operator-6dbd475d79-shg67 -n pxc-operator -- curl -s http://version-service.smart-update1-22666.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NYsiq1xBS2
++ cat /tmp/tmp.mpmV3k3SDC
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.NYsiq1xBS2 /tmp/tmp.mpmV3k3SDC
++ return 0
+ IMAGE_PXC=percona/percona-xtradb-cluster:8.0.19-10.1
+ desc 'patch crd'
+ set +o xtrace


-----------------------------------------------------------------------------------
patch crd
-----------------------------------------------------------------------------------

+ kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.NpK1xYyhbu
++ mktemp
+ local LAST_ERR=/tmp/tmp.zdL1mWScgp
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.NpK1xYyhbu
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched
+ cat /tmp/tmp.zdL1mWScgp
+ rm /tmp/tmp.NpK1xYyhbu /tmp/tmp.zdL1mWScgp
+ return 0
+ desc 'Updating ProxySQL PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Updating ProxySQL PXC cluster
-----------------------------------------------------------------------------------

+ cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/conf/smart-update.yml /tmp/tmp.zjfOka2qOE/smart-update.yml
+ yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1522-1ed4605b"' /tmp/tmp.zjfOka2qOE/smart-update.yml
+ spinup_pxc smart-update /tmp/tmp.zjfOka2qOE/smart-update.yml
+ local cluster=smart-update
+ local config=/tmp/tmp.zjfOka2qOE/smart-update.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.pTMDEDkKl6
++ mktemp
+ local LAST_ERR=/tmp/tmp.pZ8rNjCfxx
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.pTMDEDkKl6
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.pZ8rNjCfxx
+ rm /tmp/tmp.pTMDEDkKl6 /tmp/tmp.pZ8rNjCfxx
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ local LAST_OUT=/tmp/tmp.K9AZR110XP
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:k8spxc-1313-8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1522-1ed4605b#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-22666~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ local LAST_ERR=/tmp/tmp.WQpucolFEX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.K9AZR110XP
deployment.apps/pxc-client created
+ cat /tmp/tmp.WQpucolFEX
+ rm /tmp/tmp.K9AZR110XP /tmp/tmp.WQpucolFEX
+ return 0
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ apply_config /tmp/tmp.zjfOka2qOE/smart-update.yml
+ '[' -z '' ']'
+ cat_config /tmp/tmp.zjfOka2qOE/smart-update.yml
+ kubectl_bin apply -f -
+ cat /tmp/tmp.zjfOka2qOE/smart-update.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:k8spxc-1313-8.0-backup#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1522-1ed4605b#'
+ local LAST_OUT=/tmp/tmp.UzlNCN7aq7
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update1-22666~
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ local LAST_ERR=/tmp/tmp.I0QizwnY8b
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UzlNCN7aq7
perconaxtradbcluster.pxc.percona.com/smart-update created
+ cat /tmp/tmp.I0QizwnY8b
+ rm /tmp/tmp.UzlNCN7aq7 /tmp/tmp.I0QizwnY8b
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy smart-update
++ local target_cluster=smart-update
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.v9J7DHbMeg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.WIqFJZpYQS
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.v9J7DHbMeg
+++ cat /tmp/tmp.WIqFJZpYQS
+++ rm /tmp/tmp.v9J7DHbMeg /tmp/tmp.WIqFJZpYQS
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.yIjDmQ3uI0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.JrR8NYfcKj
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.yIjDmQ3uI0
+++ cat /tmp/tmp.JrR8NYfcKj
+++ rm /tmp/tmp.yIjDmQ3uI0 /tmp/tmp.JrR8NYfcKj
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo smart-update-proxysql
++ return
+ local proxy=smart-update-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-22666
++ mktemp
+ local LAST_OUT=/tmp/tmp.Y4xTQ49Qst
++ mktemp
+ local LAST_ERR=/tmp/tmp.1KNosV7Zya
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-22666
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-22666
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update1-22666
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.Y4xTQ49Qst
+ cat /tmp/tmp.1KNosV7Zya
error: no matching resources found
+ rm /tmp/tmp.Y4xTQ49Qst /tmp/tmp.1KNosV7Zya
+ return 1
+ true
+ wait_for_running smart-update-proxysql 1
+ local name=smart-update-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-proxysql-0 480
+ local pod=smart-update-proxysql-0
+ local max_retry=480
+ local ns=
++ echo smart-update-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
Error from server (NotFound): pods "smart-update-proxysql-0" not found
smart-update-proxysql-0.............Ok
+ wait_for_running smart-update-pxc 3
+ local name=smart-update-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-0 480
+ local pod=smart-update-pxc-0
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-0 condition met
smart-update-pxc-0.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-1 480
+ local pod=smart-update-pxc-1
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-1 condition met
smart-update-pxc-1.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod smart-update-pxc-2 480
+ local pod=smart-update-pxc-2
+ local max_retry=480
+ local ns=
++ echo smart-update-pxc-2
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/smart-update-pxc-2 condition met
smart-update-pxc-2.Ok
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-proxysql -uroot -proot_password -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h smart-update-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RKgSrWe4C3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jrB5NfnKjo
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RKgSrWe4C3
++ cat /tmp/tmp.jrB5NfnKjo
++ rm /tmp/tmp.RKgSrWe4C3 /tmp/tmp.jrB5NfnKjo
++ return 0
+ client_pod=pxc-client-599db5b9bc-92znf
+ wait_pod pxc-client-599db5b9bc-92znf
+ local pod=pxc-client-599db5b9bc-92znf
+ local max_retry=480
+ local ns=
++ echo pxc-client-599db5b9bc-92znf
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-599db5b9bc-92znf condition met
pxc-client-599db5b9bc-92znf.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-proxysql -uroot -proot_password -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h smart-update-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YBeNA0X8oG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vtLuJyLlXH
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YBeNA0X8oG
++ cat /tmp/tmp.vtLuJyLlXH
++ rm /tmp/tmp.YBeNA0X8oG /tmp/tmp.vtLuJyLlXH
++ return 0
+ client_pod=pxc-client-599db5b9bc-92znf
+ wait_pod pxc-client-599db5b9bc-92znf
+ local pod=pxc-client-599db5b9bc-92znf
+ local max_retry=480
+ local ns=
++ echo pxc-client-599db5b9bc-92znf
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-599db5b9bc-92znf condition met
pxc-client-599db5b9bc-92znf.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Z5gpnuD8PM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SiD8QcXDSD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Z5gpnuD8PM
++ cat /tmp/tmp.SiD8QcXDSD
++ rm /tmp/tmp.Z5gpnuD8PM /tmp/tmp.SiD8QcXDSD
++ return 0
+ client_pod=pxc-client-599db5b9bc-92znf
+ wait_pod pxc-client-599db5b9bc-92znf
+ local pod=pxc-client-599db5b9bc-92znf
+ local max_retry=480
+ local ns=
++ echo pxc-client-599db5b9bc-92znf
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-599db5b9bc-92znf condition met
pxc-client-599db5b9bc-92znf.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.zjfOka2qOE/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.zjfOka2qOE/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.chAQ36kz2s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZEzkUFmfCy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.chAQ36kz2s
++ cat /tmp/tmp.ZEzkUFmfCy
++ rm /tmp/tmp.chAQ36kz2s /tmp/tmp.ZEzkUFmfCy
++ return 0
+ client_pod=pxc-client-599db5b9bc-92znf
+ wait_pod pxc-client-599db5b9bc-92znf
+ local pod=pxc-client-599db5b9bc-92znf
+ local max_retry=480
+ local ns=
++ echo pxc-client-599db5b9bc-92znf
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-599db5b9bc-92znf condition met
pxc-client-599db5b9bc-92znf.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.zjfOka2qOE/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.zjfOka2qOE/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/compare/select-1.sql
+ [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IkHhKi90wH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UWuc77PUH9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IkHhKi90wH
++ cat /tmp/tmp.UWuc77PUH9
++ rm /tmp/tmp.IkHhKi90wH /tmp/tmp.UWuc77PUH9
++ return 0
+ client_pod=pxc-client-599db5b9bc-92znf
+ wait_pod pxc-client-599db5b9bc-92znf
+ local pod=pxc-client-599db5b9bc-92znf
+ local max_retry=480
+ local ns=
++ echo pxc-client-599db5b9bc-92znf
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-599db5b9bc-92znf condition met
pxc-client-599db5b9bc-92znf.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.zjfOka2qOE/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/smart-update1/compare/select-1.sql /tmp/tmp.zjfOka2qOE/select-1.sql
++ is_keyring_plugin_in_use smart-update
++ local cluster=smart-update
++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wyjZGPcqR6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YvnEi5h5SA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wyjZGPcqR6
++ cat /tmp/tmp.YvnEi5h5SA
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.wyjZGPcqR6 /tmp/tmp.YvnEi5h5SA
++ return 0
+ '[' '' ']'
+++ get_proxy smart-update
+++ local target_cluster=smart-update
++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.qunuOFe45S
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.OBgwBoIVeB
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.qunuOFe45S
++++ cat /tmp/tmp.OBgwBoIVeB
++++ rm /tmp/tmp.qunuOFe45S /tmp/tmp.OBgwBoIVeB
++++ return 0
+++ [[ '' == \t\r\u\e ]]
++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.K6J5F0tOVo
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.OhyOYRyq9S
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get pxc smart-update -o 'jsonpath={.spec.proxysql.enabled}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.K6J5F0tOVo
++++ cat /tmp/tmp.OhyOYRyq9S
++++ rm /tmp/tmp.K6J5F0tOVo /tmp/tmp.OhyOYRyq9S
++++ return 0
+++ [[ true == \t\r\u\e ]]
+++ echo smart-update-proxysql
+++ return
++ get_proxy_primary '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0
++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password'
++ local pod=smart-update-proxysql-0
+++ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' smart-update-proxysql-0 proxysql
+++ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';'
+++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password'
+++ local pod=smart-update-proxysql-0
+++ local container_name=proxysql
+++ set +o xtrace
++ local ip=smart-update-pxc-0.smart-update-pxc.smart-update1-22666.svc.cluster.local
+++ echo smart-update-pxc-0.smart-update-pxc.smart-update1-22666.svc.cluster.local
+++ wc -l
++ '[' 1 '!=' 1 ']'
++ echo smart-update-pxc-0.smart-update-pxc.smart-update1-22666.svc.cluster.local
++ cut -d. -f1
+ initial_primary=smart-update-pxc-0
+ kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.n40anqDWWM
++ mktemp
+ local LAST_ERR=/tmp/tmp.jM124cCyTj
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc/smart-update --type=merge -p '{"spec":{"pxc":{"image":"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.n40anqDWWM
perconaxtradbcluster.pxc.percona.com/smart-update patched
+ cat /tmp/tmp.jM124cCyTj
+ rm /tmp/tmp.n40anqDWWM /tmp/tmp.jM124cCyTj
+ return 0
+ sleep 7
+ desc 'check last pod to update'
+ set +o xtrace


-----------------------------------------------------------------------------------
check last pod to update
-----------------------------------------------------------------------------------

+ check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ local cluster=smart-update
+ local initial_primary=smart-update-pxc-0
+ local pxc_size=3
+ local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0
+ set +x
Waiting for the last pod to update..............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................