Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/logs/demand-backup-cloud-8-0.log
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ main
+ '[' -n '' ']'
+ create_infra demand-backup-cloud-29691
+ local ns=demand-backup-cloud-29691
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n demand-backup-cloud-11744 demand-backup-cloud --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/demand-backup-cloud patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.tb85nWquVn
++ mktemp
+ local LAST_ERR=/tmp/tmp.HIi3hwiOKI
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.tb85nWquVn
perconaxtradbcluster.pxc.percona.com "demand-backup-cloud" deleted from demand-backup-cloud-11744 namespace
+ cat /tmp/tmp.HIi3hwiOKI
+ rm /tmp/tmp.tb85nWquVn /tmp/tmp.HIi3hwiOKI
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.idayTS2EUD
++ mktemp
+ local LAST_ERR=/tmp/tmp.qwzEPIrHku
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.idayTS2EUD
+ cat /tmp/tmp.qwzEPIrHku
error: the server doesn't have a resource type "pxc-backup"
+ rm /tmp/tmp.idayTS2EUD /tmp/tmp.qwzEPIrHku
+ return 1
+ :
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.ehYlrXYDSO
++ mktemp
+ local LAST_ERR=/tmp/tmp.blUU74hWQh
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.ehYlrXYDSO
+ cat /tmp/tmp.blUU74hWQh
error: the server doesn't have a resource type "pxc-restore"
+ rm /tmp/tmp.ehYlrXYDSO /tmp/tmp.blUU74hWQh
+ return 1
+ :
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl api-resources
++ grep chaos-mesh.org
++ kubectl get crd
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ xargs kubectl delete ns
+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.0afLpvvYcP
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ew0G6czXAg
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.XqIiCWTTJd
++ mktemp
+ local LAST_ERR=/tmp/tmp.3NG6NglC8n
+ local exit_status=0
+ awk '{print$1}'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.0afLpvvYcP
+ cat /tmp/tmp.Ew0G6czXAg
+ rm /tmp/tmp.0afLpvvYcP /tmp/tmp.Ew0G6czXAg
+ return 0
namespace "demand-backup-cloud-11744" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XqIiCWTTJd
namespace "pxc-operator" deleted
+ cat /tmp/tmp.3NG6NglC8n
+ rm /tmp/tmp.XqIiCWTTJd /tmp/tmp.3NG6NglC8n
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z5yznhVscP
++ mktemp
+ local LAST_ERR=/tmp/tmp.gIBTRzviKE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Z5yznhVscP
namespace/pxc-operator created
+ cat /tmp/tmp.gIBTRzviKE
+ rm /tmp/tmp.Z5yznhVscP /tmp/tmp.gIBTRzviKE
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X59TNnnPp6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YKDO0FTwNh
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.X59TNnnPp6
++ cat /tmp/tmp.YKDO0FTwNh
++ rm /tmp/tmp.X59TNnnPp6 /tmp/tmp.YKDO0FTwNh
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster5 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.47wkgvp2uJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.QOInH6IGr8
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster5 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.47wkgvp2uJ
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster5" modified.
+ cat /tmp/tmp.QOInH6IGr8
+ rm /tmp/tmp.47wkgvp2uJ /tmp/tmp.QOInH6IGr8
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.s8BYjeGpRn
++ mktemp
+ local LAST_ERR=/tmp/tmp.2FOCUu0DbT
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.s8BYjeGpRn
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.2FOCUu0DbT
+ rm /tmp/tmp.s8BYjeGpRn /tmp/tmp.2FOCUu0DbT
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.xGIMyzQdff
++ mktemp
+ local LAST_ERR=/tmp/tmp.G0u922fGEk
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xGIMyzQdff
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.G0u922fGEk
+ rm /tmp/tmp.xGIMyzQdff /tmp/tmp.G0u922fGEk
+ return 0
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/cw-operator.yaml
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.rcDtiIjzuF
++ mktemp
+ local LAST_ERR=/tmp/tmp.tuh75IUywG
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.rcDtiIjzuF
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.tuh75IUywG
+ rm /tmp/tmp.rcDtiIjzuF /tmp/tmp.tuh75IUywG
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.HqQwtCe9KP
++ mktemp
+ local LAST_ERR=/tmp/tmp.BA8YSqFA4Y
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.HqQwtCe9KP
pod/percona-xtradb-cluster-operator-859595f865-cv7vl condition met
+ cat /tmp/tmp.BA8YSqFA4Y
+ rm /tmp/tmp.HqQwtCe9KP /tmp/tmp.BA8YSqFA4Y
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XdpRxGlxuD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qWZ6TR7tqR
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XdpRxGlxuD
++ cat /tmp/tmp.qWZ6TR7tqR
++ rm /tmp/tmp.XdpRxGlxuD /tmp/tmp.qWZ6TR7tqR
++ return 0
+ wait_pod percona-xtradb-cluster-operator-859595f865-cv7vl 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-859595f865-cv7vl
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-859595f865-cv7vl
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-859595f865-cv7vl condition met
waiting for pod/percona-xtradb-cluster-operator-859595f865-cv7vl to become Ready.Ok
+ sleep 3
+ create_namespace demand-backup-cloud-29691
+ local namespace=demand-backup-cloud-29691
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-cloud-29691'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-cloud-29691
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-cloud-29691
++ mktemp
++ mktemp
+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ local LAST_OUT=/tmp/tmp.NGCzuAQuSL
+ local LAST_OUT=/tmp/tmp.FzdHEdaSns
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.kERX7Rgblk
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.RvGPSCx2pF
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-cloud-29691
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-cloud-29691
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.FzdHEdaSns
+ cat /tmp/tmp.RvGPSCx2pF
+ rm /tmp/tmp.FzdHEdaSns /tmp/tmp.RvGPSCx2pF
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-cloud-29691
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.NGCzuAQuSL
+ cat /tmp/tmp.kERX7Rgblk
Error from server (NotFound): namespaces "demand-backup-cloud-29691" not found
+ rm /tmp/tmp.NGCzuAQuSL /tmp/tmp.kERX7Rgblk
+ return 1
+ :
+ wait_for_delete namespace/demand-backup-cloud-29691
+ local res=namespace/demand-backup-cloud-29691
+ echo -n 'waiting for namespace/demand-backup-cloud-29691 to be deleted'
waiting for namespace/demand-backup-cloud-29691 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "demand-backup-cloud-29691" not found
+ desc 'create namespace demand-backup-cloud-29691'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-cloud-29691
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-cloud-29691
++ mktemp
+ local LAST_OUT=/tmp/tmp.gvZiOiMhIK
++ mktemp
+ local LAST_ERR=/tmp/tmp.87ufSZNLFy
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace demand-backup-cloud-29691
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gvZiOiMhIK
namespace/demand-backup-cloud-29691 created
+ cat /tmp/tmp.87ufSZNLFy
+ rm /tmp/tmp.gvZiOiMhIK /tmp/tmp.87ufSZNLFy
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nM9G3MAZdi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZOiUR38hfk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nM9G3MAZdi
++ cat /tmp/tmp.ZOiUR38hfk
++ rm /tmp/tmp.nM9G3MAZdi /tmp/tmp.ZOiUR38hfk
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster5 --namespace=demand-backup-cloud-29691
++ mktemp
+ local LAST_OUT=/tmp/tmp.coEMpLbDxW
++ mktemp
+ local LAST_ERR=/tmp/tmp.QximpHCoIT
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster5 --namespace=demand-backup-cloud-29691
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.coEMpLbDxW
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster5" modified.
+ cat /tmp/tmp.QximpHCoIT
+ rm /tmp/tmp.coEMpLbDxW /tmp/tmp.QximpHCoIT
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.LwN9zLwPeT
++ mktemp
+ local LAST_ERR=/tmp/tmp.INSN73rRKQ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.LwN9zLwPeT
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.INSN73rRKQ
+ rm /tmp/tmp.LwN9zLwPeT /tmp/tmp.INSN73rRKQ
+ return 0
+ cluster=demand-backup-cloud
+ spinup_pxc demand-backup-cloud /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/conf/demand-backup-cloud.yml
+ local cluster=demand-backup-cloud
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/conf/demand-backup-cloud.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.CBUMdV3vU2
++ mktemp
+ local LAST_ERR=/tmp/tmp.2bOnDM8vu0
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.CBUMdV3vU2
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.2bOnDM8vu0
+ rm /tmp/tmp.CBUMdV3vU2 /tmp/tmp.2bOnDM8vu0
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ local LAST_OUT=/tmp/tmp.MQ2PD9we3m
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
++ mktemp
+ local LAST_ERR=/tmp/tmp.0SWo9zfuWG
+ local exit_status=0
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.demand-backup-cloud-29691~
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7#'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MQ2PD9we3m
deployment.apps/pxc-client created
+ cat /tmp/tmp.0SWo9zfuWG
+ rm /tmp/tmp.MQ2PD9we3m /tmp/tmp.0SWo9zfuWG
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/conf/demand-backup-cloud.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/conf/demand-backup-cloud.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/conf/demand-backup-cloud.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ local LAST_OUT=/tmp/tmp.dlKvkmZqqv
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
++ mktemp
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.demand-backup-cloud-29691~
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ local LAST_ERR=/tmp/tmp.d9feJJXaWb
+ local exit_status=0
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.dlKvkmZqqv
perconaxtradbcluster.pxc.percona.com/demand-backup-cloud created
+ cat /tmp/tmp.d9feJJXaWb
+ rm /tmp/tmp.dlKvkmZqqv /tmp/tmp.d9feJJXaWb
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy demand-backup-cloud
++ local target_cluster=demand-backup-cloud
+++ kubectl_bin get pxc demand-backup-cloud -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Zsmak7QHJ0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cmiqLxShU4
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc demand-backup-cloud -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.Zsmak7QHJ0
+++ cat /tmp/tmp.cmiqLxShU4
+++ rm /tmp/tmp.Zsmak7QHJ0 /tmp/tmp.cmiqLxShU4
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc demand-backup-cloud -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.iS55Su9lSO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5NS2yfZbT6
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc demand-backup-cloud -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.iS55Su9lSO
+++ cat /tmp/tmp.5NS2yfZbT6
+++ rm /tmp/tmp.iS55Su9lSO /tmp/tmp.5NS2yfZbT6
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo demand-backup-cloud-proxysql
++ return
+ local proxy=demand-backup-cloud-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-cloud-29691
++ mktemp
+ local LAST_OUT=/tmp/tmp.tql3YowUWp
++ mktemp
+ local LAST_ERR=/tmp/tmp.XxozWuupvr
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-cloud-29691
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-cloud-29691
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-cloud-29691
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.tql3YowUWp
+ cat /tmp/tmp.XxozWuupvr
error: no matching resources found
+ rm /tmp/tmp.tql3YowUWp /tmp/tmp.XxozWuupvr
+ return 1
+ true
+ wait_for_running demand-backup-cloud-proxysql 1
+ local name=demand-backup-cloud-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod demand-backup-cloud-proxysql-0 480
+ local pod=demand-backup-cloud-proxysql-0
+ local max_retry=480
+ local ns=
++ echo demand-backup-cloud-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
Error from server (NotFound): pods "demand-backup-cloud-proxysql-0" not found
waiting for pod/demand-backup-cloud-proxysql-0 to become Ready..........Ok
+ wait_for_running demand-backup-cloud-pxc 3
+ local name=demand-backup-cloud-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod demand-backup-cloud-pxc-0 480
+ local pod=demand-backup-cloud-pxc-0
+ local max_retry=480
+ local ns=
++ echo demand-backup-cloud-pxc-0
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/demand-backup-cloud-pxc-0 condition met
waiting for pod/demand-backup-cloud-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod demand-backup-cloud-pxc-1 480
+ local pod=demand-backup-cloud-pxc-1
+ local max_retry=480
+ local ns=
++ echo demand-backup-cloud-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/demand-backup-cloud-pxc-1 condition met
waiting for pod/demand-backup-cloud-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod demand-backup-cloud-pxc-2 480
+ local pod=demand-backup-cloud-pxc-2
+ local max_retry=480
+ local ns=
++ echo demand-backup-cloud-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/demand-backup-cloud-pxc-2 condition met
waiting for pod/demand-backup-cloud-pxc-2 to become Ready.Ok
+ sleep 10
++ kubectl get pxc demand-backup-cloud -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pE0obCjuNJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wjLIjFzjl6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pE0obCjuNJ
++ cat /tmp/tmp.wjLIjFzjl6
++ rm /tmp/tmp.pE0obCjuNJ /tmp/tmp.wjLIjFzjl6
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h demand-backup-cloud-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h demand-backup-cloud-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gGcUK7YrfS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fvIPkyLfA2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gGcUK7YrfS
++ cat /tmp/tmp.fvIPkyLfA2
++ rm /tmp/tmp.gGcUK7YrfS /tmp/tmp.fvIPkyLfA2
++ return 0
+ client_pod=pxc-client-59944c5bbf-5h9xm
+ wait_pod pxc-client-59944c5bbf-5h9xm
+ local pod=pxc-client-59944c5bbf-5h9xm
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-5h9xm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-5h9xm condition met
waiting for pod/pxc-client-59944c5bbf-5h9xm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h demand-backup-cloud-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h demand-backup-cloud-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K1gLWkcqzn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FeBPPBK7th
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.K1gLWkcqzn
++ cat /tmp/tmp.FeBPPBK7th
++ rm /tmp/tmp.K1gLWkcqzn /tmp/tmp.FeBPPBK7th
++ return 0
+ client_pod=pxc-client-59944c5bbf-5h9xm
+ wait_pod pxc-client-59944c5bbf-5h9xm
+ local pod=pxc-client-59944c5bbf-5h9xm
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-5h9xm
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-5h9xm condition met
waiting for pod/pxc-client-59944c5bbf-5h9xm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h demand-backup-cloud-pxc-0.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-cloud-pxc-0.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h demand-backup-cloud-pxc-0.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-cloud-pxc-0.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NasFGCydaS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vuKcMCuXjr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NasFGCydaS
++ cat /tmp/tmp.vuKcMCuXjr
++ rm /tmp/tmp.NasFGCydaS /tmp/tmp.vuKcMCuXjr
++ return 0
+ client_pod=pxc-client-59944c5bbf-5h9xm
+ wait_pod pxc-client-59944c5bbf-5h9xm
+ local pod=pxc-client-59944c5bbf-5h9xm
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-59944c5bbf-5h9xm
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-5h9xm condition met
waiting for pod/pxc-client-59944c5bbf-5h9xm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.uYxllufFzY/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/compare/select-1.sql /tmp/tmp.uYxllufFzY/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h demand-backup-cloud-pxc-1.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-cloud-pxc-1.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h demand-backup-cloud-pxc-1.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-cloud-pxc-1.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.30Q8hOKpD6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2MxviwORvE
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.30Q8hOKpD6
++ cat /tmp/tmp.2MxviwORvE
++ rm /tmp/tmp.30Q8hOKpD6 /tmp/tmp.2MxviwORvE
++ return 0
+ client_pod=pxc-client-59944c5bbf-5h9xm
+ wait_pod pxc-client-59944c5bbf-5h9xm
+ local pod=pxc-client-59944c5bbf-5h9xm
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-5h9xm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-5h9xm condition met
waiting for pod/pxc-client-59944c5bbf-5h9xm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.uYxllufFzY/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/compare/select-1.sql /tmp/tmp.uYxllufFzY/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h demand-backup-cloud-pxc-2.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-cloud-pxc-2.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h demand-backup-cloud-pxc-2.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-cloud-pxc-2.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QwNjvKQ2iO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.I5agiKDLps
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.QwNjvKQ2iO
++ cat /tmp/tmp.I5agiKDLps
++ rm /tmp/tmp.QwNjvKQ2iO /tmp/tmp.I5agiKDLps
++ return 0
+ client_pod=pxc-client-59944c5bbf-5h9xm
+ wait_pod pxc-client-59944c5bbf-5h9xm
+ local pod=pxc-client-59944c5bbf-5h9xm
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-5h9xm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-5h9xm condition met
waiting for pod/pxc-client-59944c5bbf-5h9xm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.uYxllufFzY/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/compare/select-1.sql /tmp/tmp.uYxllufFzY/select-1.sql
++ is_keyring_plugin_in_use demand-backup-cloud
++ local cluster=demand-backup-cloud
++ kubectl_bin exec -it demand-backup-cloud-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YQZ5BlK1o4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.x5PasVRdg1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it demand-backup-cloud-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YQZ5BlK1o4
++ cat /tmp/tmp.x5PasVRdg1
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.YQZ5BlK1o4 /tmp/tmp.x5PasVRdg1
++ return 0
+ '[' '' ']'
+ backup_name_aws=on-demand-backup-aws-s3
+ backup_name_gcp=on-demand-backup-gcp-cs
+ backup_name_azure=on-demand-backup-azure-blob
+ desc 'Run backup on-demand-backup-aws-s3 for demand-backup-cloud cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Run backup on-demand-backup-aws-s3 for demand-backup-cloud cluster
-----------------------------------------------------------------------------------

+ run_backup_with_delete on-demand-backup-aws-s3
+ local backup_name=on-demand-backup-aws-s3
+ desc 'make backup on-demand-backup-aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
make backup on-demand-backup-aws-s3
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/conf/on-demand-backup-aws-s3.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.noSkhAH4xq
++ mktemp
+ local LAST_ERR=/tmp/tmp.7wPRGxY8Ub
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/conf/on-demand-backup-aws-s3.yml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/conf/on-demand-backup-aws-s3.yml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/conf/on-demand-backup-aws-s3.yml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.noSkhAH4xq
+ cat /tmp/tmp.7wPRGxY8Ub
error: resource mapping not found for name: "on-demand-backup-aws-s3" namespace: "" from "/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup-cloud/conf/on-demand-backup-aws-s3.yml": no matches for kind "PerconaXtraDBClusterBackup" in version "pxc.percona.com/v1"
ensure CRDs are installed first
+ rm /tmp/tmp.noSkhAH4xq /tmp/tmp.7wPRGxY8Ub
+ return 1