Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/logs/recreate-8-0.log
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
+ create_infra recreate-26044
+ local ns=recreate-26044
+ '[' -n pxc-operator ']'
+ read -r pns name
+ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}'
+ [[ -z recreate-11076 ]]
+ [[ -z some-name ]]
+ kubectl_bin patch pxc -n recreate-11076 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.b7KsDYPL7S
++ mktemp
+ local LAST_ERR=/tmp/tmp.LH4VAVwZxj
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc -n recreate-11076 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.b7KsDYPL7S
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.LH4VAVwZxj
+ rm /tmp/tmp.b7KsDYPL7S /tmp/tmp.LH4VAVwZxj
+ return 0
+ read -r pns name
+ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}'
+ read -r pns name
+ timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found
perconaxtradbcluster.pxc.percona.com "some-name" deleted from recreate-11076 namespace
+ timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found
No resources found
+ timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found
No resources found
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
++ helm list --all-namespaces --filter chaos-mesh
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get MutatingWebhookConfiguration
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ awk '{print $1}'
++ grep chaos-mesh.org
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrole
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ awk '{print$1}'
++ mktemp
+ xargs kubectl delete ns
+ local LAST_OUT=/tmp/tmp.WkHs7eOkRQ
+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.SqE5cLeIth
+ local exit_status=0
+ local LAST_OUT=/tmp/tmp.i4akGyC8tB
++ mktemp
+ local LAST_ERR=/tmp/tmp.BeQDSfP6sL
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.i4akGyC8tB
+ cat /tmp/tmp.BeQDSfP6sL
+ rm /tmp/tmp.i4akGyC8tB /tmp/tmp.BeQDSfP6sL
+ return 0
namespace "recreate-11076" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.WkHs7eOkRQ
namespace "pxc-operator" deleted
+ cat /tmp/tmp.SqE5cLeIth
+ rm /tmp/tmp.WkHs7eOkRQ /tmp/tmp.SqE5cLeIth
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.eF5VpVWa87
++ mktemp
+ local LAST_ERR=/tmp/tmp.LFlDoc8WuN
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.eF5VpVWa87
namespace/pxc-operator created
+ cat /tmp/tmp.LFlDoc8WuN
+ rm /tmp/tmp.eF5VpVWa87 /tmp/tmp.LFlDoc8WuN
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DDppOUZQHB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nU8DwtGTqk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DDppOUZQHB
++ cat /tmp/tmp.nU8DwtGTqk
++ rm /tmp/tmp.DDppOUZQHB /tmp/tmp.nU8DwtGTqk
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster6 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.WDF4ySS6CA
++ mktemp
+ local LAST_ERR=/tmp/tmp.lpoh1esw8I
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster6 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.WDF4ySS6CA
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster6" modified.
+ cat /tmp/tmp.lpoh1esw8I
+ rm /tmp/tmp.WDF4ySS6CA /tmp/tmp.lpoh1esw8I
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.vXIDmGq7Mj
++ mktemp
+ local LAST_ERR=/tmp/tmp.8tUMiZhYWo
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vXIDmGq7Mj
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.8tUMiZhYWo
+ rm /tmp/tmp.vXIDmGq7Mj /tmp/tmp.8tUMiZhYWo
+ return 0
+ kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.nSu9AHFkLi
++ mktemp
+ local LAST_ERR=/tmp/tmp.yzFK40gu4a
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nSu9AHFkLi
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met
+ cat /tmp/tmp.yzFK40gu4a
E0702 03:36:36.058179   31466 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusters.pxc.percona.com&resourceVersion=1782963393617531000&timeoutSeconds=385&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+ rm /tmp/tmp.nSu9AHFkLi /tmp/tmp.yzFK40gu4a
+ return 0
+ kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.vXZV4pFPFm
++ mktemp
+ local LAST_ERR=/tmp/tmp.AqUu5nur0R
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vXZV4pFPFm
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met
+ cat /tmp/tmp.AqUu5nur0R
E0702 03:36:38.204523   31715 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterbackups.pxc.percona.com&resourceVersion=1782963393617531000&timeoutSeconds=530&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+ rm /tmp/tmp.vXZV4pFPFm /tmp/tmp.AqUu5nur0R
+ return 0
+ kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.iEvaUgueFl
++ mktemp
+ local LAST_ERR=/tmp/tmp.g6OtQNJOc3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.iEvaUgueFl
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met
+ cat /tmp/tmp.g6OtQNJOc3
E0702 03:36:39.951472   32004 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterrestores.pxc.percona.com&resourceVersion=1782963398617541000&timeoutSeconds=442&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+ rm /tmp/tmp.iEvaUgueFl /tmp/tmp.g6OtQNJOc3
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-rbac.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Sp476IiVPl
++ mktemp
+ local LAST_ERR=/tmp/tmp.PDDziNRXfr
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Sp476IiVPl
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.PDDziNRXfr
+ rm /tmp/tmp.Sp476IiVPl /tmp/tmp.PDDziNRXfr
+ return 0
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-operator.yaml
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' -
+ kubectl_bin apply -f -
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a^'
++ mktemp
+ local LAST_OUT=/tmp/tmp.LTuCHDUwrX
++ mktemp
+ local LAST_ERR=/tmp/tmp.smf0gpNhlf
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.LTuCHDUwrX
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.smf0gpNhlf
+ rm /tmp/tmp.LTuCHDUwrX /tmp/tmp.smf0gpNhlf
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.K253qilNmw
++ mktemp
+ local LAST_ERR=/tmp/tmp.HxxlUGffYP
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.K253qilNmw
pod/percona-xtradb-cluster-operator-65b5d85575-kghhm condition met
+ cat /tmp/tmp.HxxlUGffYP
E0702 03:37:01.670547    2725 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-65b5d85575-kghhm&resourceVersion=1782963421156498000&timeoutSeconds=591&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+ rm /tmp/tmp.K253qilNmw /tmp/tmp.HxxlUGffYP
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ head -1
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Veul8tvQUD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fGajATEpKj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Veul8tvQUD
++ cat /tmp/tmp.fGajATEpKj
++ rm /tmp/tmp.Veul8tvQUD /tmp/tmp.fGajATEpKj
++ return 0
+ wait_pod percona-xtradb-cluster-operator-65b5d85575-kghhm 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-65b5d85575-kghhm
+ local max_retry=480
+ local ns=pxc-operator
++ grep -E '^(pxc|proxysql)$'
++ echo percona-xtradb-cluster-operator-65b5d85575-kghhm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-65b5d85575-kghhm condition met
E0702 03:37:10.031396    3990 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-65b5d85575-kghhm&resourceVersion=1782963427114374000&timeoutSeconds=570&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/percona-xtradb-cluster-operator-65b5d85575-kghhm to become Ready.Ok
+ sleep 3
+ create_namespace recreate-26044
+ local namespace=recreate-26044
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
++ helm list --all-namespaces --filter chaos-mesh
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl api-resources
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces recreate-26044'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces recreate-26044
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace recreate-26044
+ xargs kubectl delete ns
+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.T4KvLK7GOx
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.t5MmsSABUV
+ local LAST_ERR=/tmp/tmp.sZiXNKNFGJ
+ local exit_status=0
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.ksv7ldVlke
+ for i in '$(seq 0 2)'
+ local exit_status=0
+ set +e
+ kubectl delete namespace recreate-26044
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace recreate-26044
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.t5MmsSABUV
+ cat /tmp/tmp.ksv7ldVlke
+ rm /tmp/tmp.t5MmsSABUV /tmp/tmp.ksv7ldVlke
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace recreate-26044
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.T4KvLK7GOx
+ cat /tmp/tmp.sZiXNKNFGJ
Error from server (NotFound): namespaces "recreate-26044" not found
+ rm /tmp/tmp.T4KvLK7GOx /tmp/tmp.sZiXNKNFGJ
+ return 1
+ :
+ wait_for_delete namespace/recreate-26044
+ local res=namespace/recreate-26044
+ echo -n 'waiting for namespace/recreate-26044 to be deleted'
waiting for namespace/recreate-26044 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "recreate-26044" not found
+ desc 'create namespace recreate-26044'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace recreate-26044
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace recreate-26044
++ mktemp
+ local LAST_OUT=/tmp/tmp.Prvim3CcPL
++ mktemp
+ local LAST_ERR=/tmp/tmp.fXY8OkAMBp
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace recreate-26044
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Prvim3CcPL
namespace/recreate-26044 created
+ cat /tmp/tmp.fXY8OkAMBp
+ rm /tmp/tmp.Prvim3CcPL /tmp/tmp.fXY8OkAMBp
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.v9fV6MwfWq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zm94Zcw9Io
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.v9fV6MwfWq
++ cat /tmp/tmp.zm94Zcw9Io
++ rm /tmp/tmp.v9fV6MwfWq /tmp/tmp.zm94Zcw9Io
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster6 --namespace=recreate-26044
++ mktemp
+ local LAST_OUT=/tmp/tmp.Zgvazmr5QC
++ mktemp
+ local LAST_ERR=/tmp/tmp.IadhOL2edi
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster6 --namespace=recreate-26044
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Zgvazmr5QC
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster6" modified.
+ cat /tmp/tmp.IadhOL2edi
+ rm /tmp/tmp.Zgvazmr5QC /tmp/tmp.IadhOL2edi
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.knpEi6rqzd
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ga4Vk8YBR3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.knpEi6rqzd
secret/minio-secret created
secret/aws-s3-secret created
secret/do-spaces-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.Ga4Vk8YBR3
+ rm /tmp/tmp.knpEi6rqzd /tmp/tmp.Ga4Vk8YBR3
+ return 0
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ cluster=some-name
+ spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ local cluster=some-name
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.4PuiiGi6Fc
++ mktemp
+ local LAST_ERR=/tmp/tmp.h7MWIX3poY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.4PuiiGi6Fc
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.h7MWIX3poY
+ rm /tmp/tmp.4PuiiGi6Fc /tmp/tmp.h7MWIX3poY
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml
+ local pvc_name=
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml
+ local pvc_name=
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.recreate-26044~
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.w8rOhbbheC
++ mktemp
+ local LAST_ERR=/tmp/tmp.5pDwy4i7Su
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.w8rOhbbheC
deployment.apps/pxc-client created
+ cat /tmp/tmp.5pDwy4i7Su
+ rm /tmp/tmp.w8rOhbbheC /tmp/tmp.5pDwy4i7Su
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ local pvc_name=
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ kubectl_bin apply -f -
++ mktemp
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ local LAST_OUT=/tmp/tmp.feB9JVXgw0
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.recreate-26044~
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
++ mktemp
+ local LAST_ERR=/tmp/tmp.zoSdxc8OIA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.feB9JVXgw0
perconaxtradbcluster.pxc.percona.com/some-name created
+ cat /tmp/tmp.zoSdxc8OIA
+ rm /tmp/tmp.feB9JVXgw0 /tmp/tmp.zoSdxc8OIA
+ return 0
+ desc 'waiting pods to be running for some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting pods to be running for some-name
-----------------------------------------------------------------------------------

++ get_proxy some-name
++ local target_cluster=some-name
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.LcMos8g0Ku
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NjtMEYVFaV
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.LcMos8g0Ku
+++ cat /tmp/tmp.NjtMEYVFaV
+++ rm /tmp/tmp.LcMos8g0Ku /tmp/tmp.NjtMEYVFaV
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.C21yiDFsSA
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.flSxysodEm
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.C21yiDFsSA
+++ cat /tmp/tmp.flSxysodEm
+++ rm /tmp/tmp.C21yiDFsSA /tmp/tmp.flSxysodEm
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo some-name-proxysql
++ return
+ local proxy=some-name-proxysql
++ get_proxy_size some-name
++ local cluster=some-name
++ local size=0
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wpwCLdfCDJ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.V52LKpqxA8
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.wpwCLdfCDJ
+++ cat /tmp/tmp.V52LKpqxA8
+++ rm /tmp/tmp.wpwCLdfCDJ /tmp/tmp.V52LKpqxA8
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.rFMhCd8VNB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.E1d5EUh3zH
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.rFMhCd8VNB
+++ cat /tmp/tmp.E1d5EUh3zH
+++ rm /tmp/tmp.rFMhCd8VNB /tmp/tmp.E1d5EUh3zH
+++ return 0
++ [[ true == \t\r\u\e ]]
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.size}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.XEtUOyDneA
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.0b8h4p7xbX
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.size}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.XEtUOyDneA
+++ cat /tmp/tmp.0b8h4p7xbX
+++ rm /tmp/tmp.XEtUOyDneA /tmp/tmp.0b8h4p7xbX
+++ return 0
++ size=2
++ echo 2
+ local proxy_size=2
+ wait_for_running some-name-pxc 3
+ local name=some-name-pxc
+ local size=3
+ local last_pod
+ local max_retry=480
+ [[ -z some-name-pxc ]]
+ [[ 3 -le 0 ]]
+ last_pod=2
+ desc 'Waiting for 3 running pods for some-name-pxc'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 3 running pods for some-name-pxc
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-0 480
+ local pod=some-name-pxc-0
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ echo some-name-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-0 condition met
waiting for pod/some-name-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-1 480
+ local pod=some-name-pxc-1
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-1 condition met
waiting for pod/some-name-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-2 480
+ local pod=some-name-pxc-2
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-2 condition met
waiting for pod/some-name-pxc-2 to become Ready.Ok
+ wait_for_running some-name-proxysql 2
+ local name=some-name-proxysql
+ local size=2
+ local last_pod
+ local max_retry=480
+ [[ -z some-name-proxysql ]]
+ [[ 2 -le 0 ]]
+ last_pod=1
+ desc 'Waiting for 2 running pods for some-name-proxysql'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 2 running pods for some-name-proxysql
-----------------------------------------------------------------------------------

++ seq 0 1
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-proxysql-0 480
+ local pod=some-name-proxysql-0
+ local max_retry=480
+ local ns=
++ echo some-name-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
pod/some-name-proxysql-0 condition met
E0702 03:42:22.567368   17436 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-proxysql-0&resourceVersion=1782963738175014000&timeoutSeconds=579&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/some-name-proxysql-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-proxysql-1 480
+ local pod=some-name-proxysql-1
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ echo some-name-proxysql-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=proxysql
+ set +o xtrace
pod/some-name-proxysql-1 condition met
E0702 03:42:30.583862   18660 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-proxysql-1&resourceVersion=1782963748175881000&timeoutSeconds=521&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/some-name-proxysql-1 to become Ready.Ok
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local pxc_size=3
+ local proxy_size=2
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local max_seconds=1500
+ local delay=5
+ local retries=300
+ local pxc_ready
+ local proxy_ready
+ sleep 7
+ echo -n 'waiting for pxc/some-name to be ready'
waiting for pxc/some-name to be ready++ seq 1 300
+ for _ in '$(seq 1 "${retries}")'
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pNwIzmzm1q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1po5pJwaxP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pNwIzmzm1q
++ cat /tmp/tmp.1po5pJwaxP
++ rm /tmp/tmp.pNwIzmzm1q /tmp/tmp.1po5pJwaxP
++ return 0
+ state=ready
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XvZrWhkde8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mtwlFivQ2e
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XvZrWhkde8
++ cat /tmp/tmp.mtwlFivQ2e
++ rm /tmp/tmp.XvZrWhkde8 /tmp/tmp.mtwlFivQ2e
++ return 0
+ pxc_ready=3
+ proxy_ready=0
+ [[ 2 -gt 0 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.1MJcWUBmbw
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.lm6H3pLL36
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.1MJcWUBmbw
+++++ cat /tmp/tmp.lm6H3pLL36
+++++ rm /tmp/tmp.1MJcWUBmbw /tmp/tmp.lm6H3pLL36
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.23ekHHAPQ7
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.k0gvpgN0jG
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.23ekHHAPQ7
+++++ cat /tmp/tmp.k0gvpgN0jG
+++++ rm /tmp/tmp.23ekHHAPQ7 /tmp/tmp.k0gvpgN0jG
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VjHcULwMrH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0cwPKTffuF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VjHcULwMrH
++ cat /tmp/tmp.0cwPKTffuF
++ rm /tmp/tmp.VjHcULwMrH /tmp/tmp.0cwPKTffuF
++ return 0
+ proxy_ready=2
+ [[ ready == \r\e\a\d\y ]]
+ [[ 3 == \3 ]]
+ [[ 2 == \2 ]]
+ echo

+ return
+ sleep 10
++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ base64 --decode
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3DnjbHq7gR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0CTElitGo0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3DnjbHq7gR
++ cat /tmp/tmp.0CTElitGo0
++ rm /tmp/tmp.3DnjbHq7gR /tmp/tmp.0CTElitGo0
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vgqcaQ4jrk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YKJQeSoiz3
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vgqcaQ4jrk
++ cat /tmp/tmp.YKJQeSoiz3
++ rm /tmp/tmp.vgqcaQ4jrk /tmp/tmp.YKJQeSoiz3
++ return 0
+ client_pod=pxc-client-67fc4995bb-w6jmh
+ wait_pod pxc-client-67fc4995bb-w6jmh
+ local pod=pxc-client-67fc4995bb-w6jmh
+ local max_retry=480
+ local ns=
++ echo pxc-client-67fc4995bb-w6jmh
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-w6jmh condition met
E0702 03:43:08.761313   24614 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w6jmh&resourceVersion=1782963787702121000&timeoutSeconds=583&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/pxc-client-67fc4995bb-w6jmh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ write_data some-name 100500 no-proxy -P3306
+ local cluster=some-name
+ local id=100500
+ local use_proxy=no-proxy
+ local port=-P3306
+ local database=myApp
+ local table=myApp
+ local root_pass
++ get_password some-name root
++ local cluster=some-name
++ local user=root
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9IEE9aijeo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.aJvbYK1xa4
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.9IEE9aijeo
+++ cat /tmp/tmp.aJvbYK1xa4
+++ rm /tmp/tmp.9IEE9aijeo /tmp/tmp.aJvbYK1xa4
+++ return 0
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JESQpgpXpB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gwuTEFKj36
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.JESQpgpXpB
++ cat /tmp/tmp.gwuTEFKj36
++ rm /tmp/tmp.JESQpgpXpB /tmp/tmp.gwuTEFKj36
++ return 0
+ root_pass=root_password
++ get_proxy some-name
++ local target_cluster=some-name
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DyrhLvMCCM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9n9EqEWhal
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.DyrhLvMCCM
+++ cat /tmp/tmp.9n9EqEWhal
+++ rm /tmp/tmp.DyrhLvMCCM /tmp/tmp.9n9EqEWhal
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.bmxwVNtMIY
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.gDeCJbWBIn
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.bmxwVNtMIY
+++ cat /tmp/tmp.gDeCJbWBIn
+++ rm /tmp/tmp.bmxwVNtMIY /tmp/tmp.gDeCJbWBIn
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo some-name-proxysql
++ return
+ proxy=some-name-proxysql
+ [[ no-proxy == true ]]
+ [[ no-proxy == \u\s\e\-\p\r\o\x\y ]]
+ host='-h some-name-pxc-0.some-name-pxc'
+ run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sir6gbH1kd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tS8q0JzVVZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sir6gbH1kd
++ cat /tmp/tmp.tS8q0JzVVZ
++ rm /tmp/tmp.sir6gbH1kd /tmp/tmp.tS8q0JzVVZ
++ return 0
+ client_pod=pxc-client-67fc4995bb-w6jmh
+ wait_pod pxc-client-67fc4995bb-w6jmh
+ local pod=pxc-client-67fc4995bb-w6jmh
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-67fc4995bb-w6jmh
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-w6jmh condition met
E0702 03:43:26.593498   27058 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w6jmh&resourceVersion=1782963804039749000&timeoutSeconds=555&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/pxc-client-67fc4995bb-w6jmh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
+ desc 'compare initial data'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare initial data
-----------------------------------------------------------------------------------

+ compare_data some-name select-1 no-proxy -P3306
+ local cluster=some-name
+ local compare=select-1
+ local use_proxy=no-proxy
+ local port=-P3306
+ local database=myApp
+ local table=myApp
+ local size
+ local root_pass
++ get_password some-name root
++ local cluster=some-name
++ local user=root
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.P6JCqhDloK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.qQA2K59YNx
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.P6JCqhDloK
+++ cat /tmp/tmp.qQA2K59YNx
+++ rm /tmp/tmp.P6JCqhDloK /tmp/tmp.qQA2K59YNx
+++ return 0
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
+++ mktemp
++ base64 --decode
++ local LAST_OUT=/tmp/tmp.89KNZxAANz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IlalOThrT9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.89KNZxAANz
++ cat /tmp/tmp.IlalOThrT9
++ rm /tmp/tmp.89KNZxAANz /tmp/tmp.IlalOThrT9
++ return 0
+ root_pass=root_password
+ [[ no-proxy == true ]]
+ [[ no-proxy == \u\s\e\-\p\r\o\x\y ]]
++ get_pxc_size some-name
++ local cluster=some-name
++ local size=0
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.pxc.size}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.QXVuIGsaJH
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.yFh0Z3JK1W
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.pxc.size}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.QXVuIGsaJH
+++ cat /tmp/tmp.yFh0Z3JK1W
+++ rm /tmp/tmp.QXVuIGsaJH /tmp/tmp.yFh0Z3JK1W
+++ return 0
++ size=3
++ echo 3
+ size=3
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
+ local command_id=select-1
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Sj3kvZkWm5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hqlkNVrCut
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Sj3kvZkWm5
++ cat /tmp/tmp.hqlkNVrCut
++ rm /tmp/tmp.Sj3kvZkWm5 /tmp/tmp.hqlkNVrCut
++ return 0
+ client_pod=pxc-client-67fc4995bb-w6jmh
+ wait_pod pxc-client-67fc4995bb-w6jmh
+ local pod=pxc-client-67fc4995bb-w6jmh
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ echo pxc-client-67fc4995bb-w6jmh
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-w6jmh condition met
E0702 03:44:12.294626    1746 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w6jmh&resourceVersion=1782963850404530000&timeoutSeconds=461&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/pxc-client-67fc4995bb-w6jmh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.LREFv34t5b/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.LREFv34t5b/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.LREFv34t5b/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-1.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
+ local command_id=select-1
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-1.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GtWMfsuIYC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9bOr0pVsYN
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GtWMfsuIYC
++ cat /tmp/tmp.9bOr0pVsYN
++ rm /tmp/tmp.GtWMfsuIYC /tmp/tmp.9bOr0pVsYN
++ return 0
+ client_pod=pxc-client-67fc4995bb-w6jmh
+ wait_pod pxc-client-67fc4995bb-w6jmh
+ local pod=pxc-client-67fc4995bb-w6jmh
+ local max_retry=480
+ local ns=
++ echo pxc-client-67fc4995bb-w6jmh
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-w6jmh condition met
E0702 03:44:23.720661    3559 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w6jmh&resourceVersion=1782963861573015000&timeoutSeconds=582&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/pxc-client-67fc4995bb-w6jmh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.LREFv34t5b/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.LREFv34t5b/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.LREFv34t5b/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-2.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
+ local command_id=select-1
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-2.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
+ local 'command=SELECT id FROM myApp.myApp ORDER BY id;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -P3306 -uroot -p'\''root_password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.si70UIDEfG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LmNejlYGQs
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.si70UIDEfG
++ cat /tmp/tmp.LmNejlYGQs
++ rm /tmp/tmp.si70UIDEfG /tmp/tmp.LmNejlYGQs
++ return 0
+ client_pod=pxc-client-67fc4995bb-w6jmh
+ wait_pod pxc-client-67fc4995bb-w6jmh
+ local pod=pxc-client-67fc4995bb-w6jmh
+ local max_retry=480
+ local ns=
++ echo pxc-client-67fc4995bb-w6jmh
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-w6jmh condition met
E0702 03:44:33.852165    5172 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w6jmh&resourceVersion=1782963871945540000&timeoutSeconds=413&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/pxc-client-67fc4995bb-w6jmh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.LREFv34t5b/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.LREFv34t5b/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.LREFv34t5b/select-1.sql
+ is_keyring_plugin_in_use some-name
+ local cluster=some-name
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ grep -E -o 'early-plugin-load=keyring_\w+.so'
+ kubectl exec some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
+ return 1
+ desc pause
+ set +o xtrace


-----------------------------------------------------------------------------------
pause
-----------------------------------------------------------------------------------

+ sed -e 's/pause: false/pause: true/'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ local pvc_name=
+ kubectl_bin apply -f-
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.recreate-26044~
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
++ mktemp
+ local LAST_OUT=/tmp/tmp.z4sRmSkFXi
++ mktemp
+ local LAST_ERR=/tmp/tmp.2jxRLmvlfS
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f-
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.z4sRmSkFXi
perconaxtradbcluster.pxc.percona.com/some-name configured
+ cat /tmp/tmp.2jxRLmvlfS
+ rm /tmp/tmp.z4sRmSkFXi /tmp/tmp.2jxRLmvlfS
+ return 0
+ wait_for_delete pod/some-name-proxysql-0
+ local res=pod/some-name-proxysql-0
+ echo -n 'waiting for pod/some-name-proxysql-0 to be deleted'
waiting for pod/some-name-proxysql-0 to be deleted+ set +o xtrace
.Error from server (NotFound): pods "some-name-proxysql-0" not found
+ wait_for_delete pod/some-name-pxc-2
+ local res=pod/some-name-pxc-2
+ echo -n 'waiting for pod/some-name-pxc-2 to be deleted'
waiting for pod/some-name-pxc-2 to be deleted+ set +o xtrace
...Error from server (NotFound): pods "some-name-pxc-2" not found
+ wait_for_delete pod/some-name-pxc-1
+ local res=pod/some-name-pxc-1
+ echo -n 'waiting for pod/some-name-pxc-1 to be deleted'
waiting for pod/some-name-pxc-1 to be deleted+ set +o xtrace
.....Error from server (NotFound): pods "some-name-pxc-1" not found
+ wait_for_delete pod/some-name-pxc-0
+ local res=pod/some-name-pxc-0
+ echo -n 'waiting for pod/some-name-pxc-0 to be deleted'
waiting for pod/some-name-pxc-0 to be deleted+ set +o xtrace
...Error from server (NotFound): pods "some-name-pxc-0" not found
+ desc 'unpause, check data'
+ set +o xtrace


-----------------------------------------------------------------------------------
unpause, check data
-----------------------------------------------------------------------------------

+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ local pvc_name=
+ kubectl_bin apply -f-
+ sed -e 's/pause: true/pause: false/'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ local LAST_OUT=/tmp/tmp.QgsU1G2dB1
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.recreate-26044~
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
++ mktemp
+ local LAST_ERR=/tmp/tmp.k3IGSwhJby
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f-
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QgsU1G2dB1
perconaxtradbcluster.pxc.percona.com/some-name configured
+ cat /tmp/tmp.k3IGSwhJby
+ rm /tmp/tmp.QgsU1G2dB1 /tmp/tmp.k3IGSwhJby
+ return 0
+ wait_for_running some-name-proxysql 1
+ local name=some-name-proxysql
+ local size=1
+ local last_pod
+ local max_retry=480
+ [[ -z some-name-proxysql ]]
+ [[ 1 -le 0 ]]
+ last_pod=0
+ desc 'Waiting for 1 running pods for some-name-proxysql'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 1 running pods for some-name-proxysql
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-proxysql-0 480
+ local pod=some-name-proxysql-0
+ local max_retry=480
+ local ns=
++ echo some-name-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
pod/some-name-proxysql-0 condition met
waiting for pod/some-name-proxysql-0 to become Ready.Ok
+ wait_for_running some-name-pxc 3
+ local name=some-name-pxc
+ local size=3
+ local last_pod
+ local max_retry=480
+ [[ -z some-name-pxc ]]
+ [[ 3 -le 0 ]]
+ last_pod=2
+ desc 'Waiting for 3 running pods for some-name-pxc'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 3 running pods for some-name-pxc
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-0 480
+ local pod=some-name-pxc-0
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo some-name-pxc-0
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-0 condition met
waiting for pod/some-name-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-1 480
+ local pod=some-name-pxc-1
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ echo some-name-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-1 condition met
waiting for pod/some-name-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-2 480
+ local pod=some-name-pxc-2
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo some-name-pxc-2
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-2 condition met
waiting for pod/some-name-pxc-2 to become Ready.Ok
+ desc 'check data'
+ set +o xtrace


-----------------------------------------------------------------------------------
check data
-----------------------------------------------------------------------------------

+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6zE8Jj7LU7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FzmnOcaaf1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6zE8Jj7LU7
++ cat /tmp/tmp.FzmnOcaaf1
++ rm /tmp/tmp.6zE8Jj7LU7 /tmp/tmp.FzmnOcaaf1
++ return 0
+ client_pod=pxc-client-67fc4995bb-w6jmh
+ wait_pod pxc-client-67fc4995bb-w6jmh
+ local pod=pxc-client-67fc4995bb-w6jmh
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo pxc-client-67fc4995bb-w6jmh
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-w6jmh condition met
E0702 03:48:11.741141    4598 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w6jmh&resourceVersion=1782964089455507000&timeoutSeconds=425&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/pxc-client-67fc4995bb-w6jmh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.LREFv34t5b/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.LREFv34t5b/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.LREFv34t5b/select-1.sql
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tPGFveV2IB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HDkDFiRxGj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.tPGFveV2IB
++ cat /tmp/tmp.HDkDFiRxGj
++ rm /tmp/tmp.tPGFveV2IB /tmp/tmp.HDkDFiRxGj
++ return 0
+ client_pod=pxc-client-67fc4995bb-w6jmh
+ wait_pod pxc-client-67fc4995bb-w6jmh
+ local pod=pxc-client-67fc4995bb-w6jmh
+ local max_retry=480
+ local ns=
++ echo pxc-client-67fc4995bb-w6jmh
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-w6jmh condition met
E0702 03:48:23.261230    6415 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w6jmh&resourceVersion=1782964101555103000&timeoutSeconds=375&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/pxc-client-67fc4995bb-w6jmh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.LREFv34t5b/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.LREFv34t5b/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.LREFv34t5b/select-1.sql
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZEbO2QCrAh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vbpuman4pi
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZEbO2QCrAh
++ cat /tmp/tmp.vbpuman4pi
++ rm /tmp/tmp.ZEbO2QCrAh /tmp/tmp.vbpuman4pi
++ return 0
+ client_pod=pxc-client-67fc4995bb-w6jmh
+ wait_pod pxc-client-67fc4995bb-w6jmh
+ local pod=pxc-client-67fc4995bb-w6jmh
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-67fc4995bb-w6jmh
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-w6jmh condition met
E0702 03:48:31.349007    7707 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w6jmh&resourceVersion=1782964110521285000&timeoutSeconds=520&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/pxc-client-67fc4995bb-w6jmh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.LREFv34t5b/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.LREFv34t5b/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.LREFv34t5b/select-1.sql
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100501)' '-h some-name-proxysql -uroot -proot_password'
+ local 'command=INSERT myApp.myApp (id) VALUES (100501)'
+ local 'uri=-h some-name-proxysql -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UvM1nSqcqI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kjnIsdILeZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.UvM1nSqcqI
++ cat /tmp/tmp.kjnIsdILeZ
++ rm /tmp/tmp.UvM1nSqcqI /tmp/tmp.kjnIsdILeZ
++ return 0
+ client_pod=pxc-client-67fc4995bb-w6jmh
+ wait_pod pxc-client-67fc4995bb-w6jmh
+ local pod=pxc-client-67fc4995bb-w6jmh
+ local max_retry=480
+ local ns=
++ echo pxc-client-67fc4995bb-w6jmh
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-w6jmh condition met
E0702 03:48:38.277803    8707 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w6jmh&resourceVersion=1782964117477559000&timeoutSeconds=554&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/pxc-client-67fc4995bb-w6jmh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local command_id=select-2
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-2-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q6bAskeu0c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.W82ViT99Xi
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.q6bAskeu0c
++ cat /tmp/tmp.W82ViT99Xi
++ rm /tmp/tmp.q6bAskeu0c /tmp/tmp.W82ViT99Xi
++ return 0
+ client_pod=pxc-client-67fc4995bb-w6jmh
+ wait_pod pxc-client-67fc4995bb-w6jmh
+ local pod=pxc-client-67fc4995bb-w6jmh
+ local max_retry=480
+ local ns=
++ echo pxc-client-67fc4995bb-w6jmh
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-w6jmh condition met
E0702 03:48:47.294103    9844 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w6jmh&resourceVersion=1782964125638695000&timeoutSeconds=485&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/pxc-client-67fc4995bb-w6jmh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.LREFv34t5b/select-2.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.LREFv34t5b/select-2.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.LREFv34t5b/select-2.sql
+ compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local command_id=select-2
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-2-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SCYugrI2AI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GfqOw86rDp
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SCYugrI2AI
++ cat /tmp/tmp.GfqOw86rDp
++ rm /tmp/tmp.SCYugrI2AI /tmp/tmp.GfqOw86rDp
++ return 0
+ client_pod=pxc-client-67fc4995bb-w6jmh
+ wait_pod pxc-client-67fc4995bb-w6jmh
+ local pod=pxc-client-67fc4995bb-w6jmh
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo pxc-client-67fc4995bb-w6jmh
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-w6jmh condition met
E0702 03:48:58.008585   11373 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w6jmh&resourceVersion=1782964135471982000&timeoutSeconds=351&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/pxc-client-67fc4995bb-w6jmh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.LREFv34t5b/select-2.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.LREFv34t5b/select-2.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.LREFv34t5b/select-2.sql
+ compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local command_id=select-2
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-2-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GhF6wQE9ub
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vLVivznYHd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GhF6wQE9ub
++ cat /tmp/tmp.vLVivznYHd
++ rm /tmp/tmp.GhF6wQE9ub /tmp/tmp.vLVivznYHd
++ return 0
+ client_pod=pxc-client-67fc4995bb-w6jmh
+ wait_pod pxc-client-67fc4995bb-w6jmh
+ local pod=pxc-client-67fc4995bb-w6jmh
+ local max_retry=480
+ local ns=
++ echo pxc-client-67fc4995bb-w6jmh
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-67fc4995bb-w6jmh condition met
E0702 03:49:08.781597   13016 reflector.go:227] "Failed to watch" err="Get \"https://136.65.189.131/api/v1/namespaces/recreate-26044/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-w6jmh&resourceVersion=1782964146268619000&timeoutSeconds=485&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/pxc-client-67fc4995bb-w6jmh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.LREFv34t5b/select-2.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.LREFv34t5b/select-2.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.LREFv34t5b/select-2.sql
+ desc 'delete cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete cluster
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ASy4lvCVE0
++ mktemp
+ local LAST_ERR=/tmp/tmp.EdLexcJanQ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ASy4lvCVE0
perconaxtradbcluster.pxc.percona.com "some-name" deleted from recreate-26044 namespace
+ cat /tmp/tmp.EdLexcJanQ
+ rm /tmp/tmp.ASy4lvCVE0 /tmp/tmp.EdLexcJanQ
+ return 0
+ wait_for_delete pod/some-name-pxc-2
+ local res=pod/some-name-pxc-2
+ echo -n 'waiting for pod/some-name-pxc-2 to be deleted'
waiting for pod/some-name-pxc-2 to be deleted+ set +o xtrace
Error from server (NotFound): pods "some-name-pxc-2" not found
+ wait_for_delete pod/some-name-pxc-1
+ local res=pod/some-name-pxc-1
+ echo -n 'waiting for pod/some-name-pxc-1 to be deleted'
waiting for pod/some-name-pxc-1 to be deleted+ set +o xtrace
Error from server (NotFound): pods "some-name-pxc-1" not found
+ wait_for_delete pod/some-name-pxc-0
+ local res=pod/some-name-pxc-0
+ echo -n 'waiting for pod/some-name-pxc-0 to be deleted'
waiting for pod/some-name-pxc-0 to be deleted+ set +o xtrace
...Error from server (NotFound): pods "some-name-pxc-0" not found
+ desc 'recreate cluster, check data'
+ set +o xtrace


-----------------------------------------------------------------------------------
recreate cluster, check data
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f-
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ local pvc_name=
+ sed -e 's/storage: 2Gi/storage: 3Gi/; s/size: 3/size: 5/; s/antiAffinityTopologyKey:.*/antiAffinityTopologyKey: none/'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.recreate-26044~
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.01u8lwYtIb
++ mktemp
+ local LAST_ERR=/tmp/tmp.X29zCNpIQb
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f-
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.01u8lwYtIb
perconaxtradbcluster.pxc.percona.com/some-name created
+ cat /tmp/tmp.X29zCNpIQb
+ rm /tmp/tmp.01u8lwYtIb /tmp/tmp.X29zCNpIQb
+ return 0
+ wait_for_running some-name-proxysql 1
+ local name=some-name-proxysql
+ local size=1
+ local last_pod
+ local max_retry=480
+ [[ -z some-name-proxysql ]]
+ [[ 1 -le 0 ]]
+ last_pod=0
+ desc 'Waiting for 1 running pods for some-name-proxysql'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 1 running pods for some-name-proxysql
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-proxysql-0 480
+ local pod=some-name-proxysql-0
+ local max_retry=480
+ local ns=
++ echo some-name-proxysql-0
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=proxysql
+ set +o xtrace
pod/some-name-proxysql-0 condition met
waiting for pod/some-name-proxysql-0 to become Ready.Ok
+ wait_for_running some-name-pxc 5
+ local name=some-name-pxc
+ local size=5
+ local last_pod
+ local max_retry=480
+ [[ -z some-name-pxc ]]
+ [[ 5 -le 0 ]]
+ last_pod=4
+ desc 'Waiting for 5 running pods for some-name-pxc'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 5 running pods for some-name-pxc
-----------------------------------------------------------------------------------

++ seq 0 4
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-0 480
+ local pod=some-name-pxc-0
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo some-name-pxc-0
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-0 condition met
waiting for pod/some-name-pxc-0 to become Ready.full cluster crash detected