Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/logs/demand-backup-8-0.log
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ main
+ create_infra demand-backup-20748
+ local ns=demand-backup-20748
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n demand-backup-753 demand-backup --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (InternalError): Internal error occurred: failed calling webhook "validationwebhook.pxc.percona.com": failed to call webhook: Post "https://percona-xtradb-cluster-operator.pxc-operator.svc:443/validate-percona-xtradbcluster?timeout=10s": no endpoints available for service "percona-xtradb-cluster-operator"
+ :
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.lbuqIgNmkj
++ mktemp
+ local LAST_ERR=/tmp/tmp.gxPEHBJ1m9
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lbuqIgNmkj
perconaxtradbcluster.pxc.percona.com "demand-backup" deleted from demand-backup-753 namespace
+ cat /tmp/tmp.gxPEHBJ1m9
+ rm /tmp/tmp.lbuqIgNmkj /tmp/tmp.gxPEHBJ1m9
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.opz97DjS11
++ mktemp
+ local LAST_ERR=/tmp/tmp.fJ066hqj4R
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.opz97DjS11
+ cat /tmp/tmp.fJ066hqj4R
error: the server doesn't have a resource type "pxc-backup"
+ rm /tmp/tmp.opz97DjS11 /tmp/tmp.fJ066hqj4R
+ return 1
+ :
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.00qu1ntAkW
++ mktemp
+ local LAST_ERR=/tmp/tmp.esYODR1YAn
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.00qu1ntAkW
+ cat /tmp/tmp.esYODR1YAn
error: the server doesn't have a resource type "pxc-restore"
+ rm /tmp/tmp.00qu1ntAkW /tmp/tmp.esYODR1YAn
+ return 1
+ :
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
++ mktemp
++ mktemp
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.fKZbkw3AAS
++ mktemp
+ local LAST_OUT=/tmp/tmp.7fZRpXp7pX
++ mktemp
+ local LAST_ERR=/tmp/tmp.U2MnsMI6wO
+ local exit_status=0
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.FMfKsWa35q
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7fZRpXp7pX
+ cat /tmp/tmp.FMfKsWa35q
+ rm /tmp/tmp.7fZRpXp7pX /tmp/tmp.FMfKsWa35q
+ return 0
namespace "demand-backup-753" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.fKZbkw3AAS
namespace "pxc-operator" deleted
+ cat /tmp/tmp.U2MnsMI6wO
+ rm /tmp/tmp.fKZbkw3AAS /tmp/tmp.U2MnsMI6wO
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.MCAP5Mle2O
++ mktemp
+ local LAST_ERR=/tmp/tmp.oiobRx3g7S
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MCAP5Mle2O
namespace/pxc-operator created
+ cat /tmp/tmp.oiobRx3g7S
+ rm /tmp/tmp.MCAP5Mle2O /tmp/tmp.oiobRx3g7S
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q4mbFJWHvA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.V3k11hAb7w
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.q4mbFJWHvA
++ cat /tmp/tmp.V3k11hAb7w
++ rm /tmp/tmp.q4mbFJWHvA /tmp/tmp.V3k11hAb7w
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster7 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.WqS55CKZbL
++ mktemp
+ local LAST_ERR=/tmp/tmp.sLkXzqrVtp
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster7 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.WqS55CKZbL
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster7" modified.
+ cat /tmp/tmp.sLkXzqrVtp
+ rm /tmp/tmp.WqS55CKZbL /tmp/tmp.sLkXzqrVtp
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.1JDnGTVQ9p
++ mktemp
+ local LAST_ERR=/tmp/tmp.fHkNSXe3p7
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.1JDnGTVQ9p
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.fHkNSXe3p7
+ rm /tmp/tmp.1JDnGTVQ9p /tmp/tmp.fHkNSXe3p7
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Sxj0yAUZ32
++ mktemp
+ local LAST_ERR=/tmp/tmp.fvJAPtEY42
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Sxj0yAUZ32
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.fvJAPtEY42
+ rm /tmp/tmp.Sxj0yAUZ32 /tmp/tmp.fvJAPtEY42
+ return 0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
+ kubectl_bin apply -f -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/cw-operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.fJgOdm7Hwh
++ mktemp
+ local LAST_ERR=/tmp/tmp.UgVyUUpal3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.fJgOdm7Hwh
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.UgVyUUpal3
+ rm /tmp/tmp.fJgOdm7Hwh /tmp/tmp.UgVyUUpal3
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.jCtpG3YOGD
++ mktemp
+ local LAST_ERR=/tmp/tmp.EiaF4q5bZF
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.jCtpG3YOGD
pod/percona-xtradb-cluster-operator-859595f865-6snwp condition met
+ cat /tmp/tmp.EiaF4q5bZF
+ rm /tmp/tmp.jCtpG3YOGD /tmp/tmp.EiaF4q5bZF
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WmBqKQyRJR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wlr5UiSpIH
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WmBqKQyRJR
++ cat /tmp/tmp.wlr5UiSpIH
++ rm /tmp/tmp.WmBqKQyRJR /tmp/tmp.wlr5UiSpIH
++ return 0
+ wait_pod percona-xtradb-cluster-operator-859595f865-6snwp 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-859595f865-6snwp
+ local max_retry=480
+ local ns=pxc-operator
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo percona-xtradb-cluster-operator-859595f865-6snwp
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-859595f865-6snwp condition met
waiting for pod/percona-xtradb-cluster-operator-859595f865-6snwp to become Ready.Ok
+ sleep 3
+ create_namespace demand-backup-20748
+ local namespace=demand-backup-20748
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-20748'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-20748
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-20748
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.0n421wpWMK
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.lbk1dh1q3N
+ local exit_status=0
++ seq 0 2
+ local LAST_OUT=/tmp/tmp.Te9ayiPvmS
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-20748
+ local LAST_ERR=/tmp/tmp.Oo1iJMyU9e
+ local exit_status=0
++ seq 0 2
+ xargs kubectl delete ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-20748
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Te9ayiPvmS
+ cat /tmp/tmp.Oo1iJMyU9e
+ rm /tmp/tmp.Te9ayiPvmS /tmp/tmp.Oo1iJMyU9e
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-20748
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.0n421wpWMK
+ cat /tmp/tmp.lbk1dh1q3N
Error from server (NotFound): namespaces "demand-backup-20748" not found
+ rm /tmp/tmp.0n421wpWMK /tmp/tmp.lbk1dh1q3N
+ return 1
+ :
+ wait_for_delete namespace/demand-backup-20748
+ local res=namespace/demand-backup-20748
+ echo -n 'waiting for namespace/demand-backup-20748 to be deleted'
waiting for namespace/demand-backup-20748 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "demand-backup-20748" not found
+ desc 'create namespace demand-backup-20748'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-20748
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-20748
++ mktemp
+ local LAST_OUT=/tmp/tmp.FEbHePrIwh
++ mktemp
+ local LAST_ERR=/tmp/tmp.1Z2reVli3p
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace demand-backup-20748
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.FEbHePrIwh
namespace/demand-backup-20748 created
+ cat /tmp/tmp.1Z2reVli3p
+ rm /tmp/tmp.FEbHePrIwh /tmp/tmp.1Z2reVli3p
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RcmHKD9M9w
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tl7Wy336j5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RcmHKD9M9w
++ cat /tmp/tmp.tl7Wy336j5
++ rm /tmp/tmp.RcmHKD9M9w /tmp/tmp.tl7Wy336j5
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster7 --namespace=demand-backup-20748
++ mktemp
+ local LAST_OUT=/tmp/tmp.EQFT0NrsFG
++ mktemp
+ local LAST_ERR=/tmp/tmp.tLH6P5TUfX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster7 --namespace=demand-backup-20748
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EQFT0NrsFG
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster7" modified.
+ cat /tmp/tmp.tLH6P5TUfX
+ rm /tmp/tmp.EQFT0NrsFG /tmp/tmp.tLH6P5TUfX
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.rKRhOa2jhn
++ mktemp
+ local LAST_ERR=/tmp/tmp.UQz5dEo6xw
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.rKRhOa2jhn
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.UQz5dEo6xw
+ rm /tmp/tmp.rKRhOa2jhn /tmp/tmp.UQz5dEo6xw
+ return 0
+ cluster=demand-backup
+ spinup_pxc demand-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/conf/demand-backup.yml
+ local cluster=demand-backup
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/conf/demand-backup.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.UDpAfsG1q0
++ mktemp
+ local LAST_ERR=/tmp/tmp.iBXTEPEQtQ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UDpAfsG1q0
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.iBXTEPEQtQ
+ rm /tmp/tmp.UDpAfsG1q0 /tmp/tmp.iBXTEPEQtQ
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.BbVQu9GPDO
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.demand-backup-20748~
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ local LAST_ERR=/tmp/tmp.rtGqGHsaNS
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.BbVQu9GPDO
deployment.apps/pxc-client created
+ cat /tmp/tmp.rtGqGHsaNS
+ rm /tmp/tmp.BbVQu9GPDO /tmp/tmp.rtGqGHsaNS
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/conf/demand-backup.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/conf/demand-backup.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/conf/demand-backup.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ local LAST_OUT=/tmp/tmp.J27dgUipdk
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.demand-backup-20748~
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ local LAST_ERR=/tmp/tmp.6p7d0APOnH
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.J27dgUipdk
perconaxtradbcluster.pxc.percona.com/demand-backup created
+ cat /tmp/tmp.6p7d0APOnH
+ rm /tmp/tmp.J27dgUipdk /tmp/tmp.6p7d0APOnH
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy demand-backup
++ local target_cluster=demand-backup
+++ kubectl_bin get pxc demand-backup -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.l4R66FkP4O
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.MXc4DHVQt5
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc demand-backup -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.l4R66FkP4O
+++ cat /tmp/tmp.MXc4DHVQt5
+++ rm /tmp/tmp.l4R66FkP4O /tmp/tmp.MXc4DHVQt5
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc demand-backup -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.nfccMO7ikH
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1GLQjtG1W3
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc demand-backup -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.nfccMO7ikH
+++ cat /tmp/tmp.1GLQjtG1W3
+++ rm /tmp/tmp.nfccMO7ikH /tmp/tmp.1GLQjtG1W3
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo demand-backup-proxysql
++ return
+ local proxy=demand-backup-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-20748
++ mktemp
+ local LAST_OUT=/tmp/tmp.ojOKrgN06u
++ mktemp
+ local LAST_ERR=/tmp/tmp.r2bdVUH5S1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-20748
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-20748
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-20748
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.ojOKrgN06u
+ cat /tmp/tmp.r2bdVUH5S1
error: no matching resources found
+ rm /tmp/tmp.ojOKrgN06u /tmp/tmp.r2bdVUH5S1
+ return 1
+ true
+ wait_for_running demand-backup-proxysql 1
+ local name=demand-backup-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod demand-backup-proxysql-0 480
+ local pod=demand-backup-proxysql-0
+ local max_retry=480
+ local ns=
++ echo demand-backup-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
Error from server (NotFound): pods "demand-backup-proxysql-0" not found
waiting for pod/demand-backup-proxysql-0 to become Ready............Ok
+ wait_for_running demand-backup-pxc 3
+ local name=demand-backup-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod demand-backup-pxc-0 480
+ local pod=demand-backup-pxc-0
+ local max_retry=480
+ local ns=
++ echo demand-backup-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/demand-backup-pxc-0 condition met
waiting for pod/demand-backup-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod demand-backup-pxc-1 480
+ local pod=demand-backup-pxc-1
+ local max_retry=480
+ local ns=
++ echo demand-backup-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/demand-backup-pxc-1 condition met
waiting for pod/demand-backup-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod demand-backup-pxc-2 480
+ local pod=demand-backup-pxc-2
+ local max_retry=480
+ local ns=
++ echo demand-backup-pxc-2
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/demand-backup-pxc-2 condition met
waiting for pod/demand-backup-pxc-2 to become Ready.Ok
+ sleep 10
++ kubectl get pxc demand-backup -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bFtdRieUWQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WCzhtbeSxf
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bFtdRieUWQ
++ cat /tmp/tmp.WCzhtbeSxf
++ rm /tmp/tmp.bFtdRieUWQ /tmp/tmp.WCzhtbeSxf
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h demand-backup-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h demand-backup-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NogUJWe5An
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MpE9UCGEcO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NogUJWe5An
++ cat /tmp/tmp.MpE9UCGEcO
++ rm /tmp/tmp.NogUJWe5An /tmp/tmp.MpE9UCGEcO
++ return 0
+ client_pod=pxc-client-59944c5bbf-4nwbf
+ wait_pod pxc-client-59944c5bbf-4nwbf
+ local pod=pxc-client-59944c5bbf-4nwbf
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-4nwbf
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-4nwbf condition met
waiting for pod/pxc-client-59944c5bbf-4nwbf to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h demand-backup-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h demand-backup-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bOBK2gMhHS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.X0OJ58qNB3
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bOBK2gMhHS
++ cat /tmp/tmp.X0OJ58qNB3
++ rm /tmp/tmp.bOBK2gMhHS /tmp/tmp.X0OJ58qNB3
++ return 0
+ client_pod=pxc-client-59944c5bbf-4nwbf
+ wait_pod pxc-client-59944c5bbf-4nwbf
+ local pod=pxc-client-59944c5bbf-4nwbf
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-4nwbf
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-4nwbf condition met
waiting for pod/pxc-client-59944c5bbf-4nwbf to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h demand-backup-pxc-0.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-pxc-0.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h demand-backup-pxc-0.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-pxc-0.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bBqq4ZpibF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Boux8ek8vl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bBqq4ZpibF
++ cat /tmp/tmp.Boux8ek8vl
++ rm /tmp/tmp.bBqq4ZpibF /tmp/tmp.Boux8ek8vl
++ return 0
+ client_pod=pxc-client-59944c5bbf-4nwbf
+ wait_pod pxc-client-59944c5bbf-4nwbf
+ local pod=pxc-client-59944c5bbf-4nwbf
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-4nwbf
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-4nwbf condition met
waiting for pod/pxc-client-59944c5bbf-4nwbf to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.Ww3sIkKqjx/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/compare/select-1.sql /tmp/tmp.Ww3sIkKqjx/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h demand-backup-pxc-1.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-pxc-1.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h demand-backup-pxc-1.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-pxc-1.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hq4gn2DqVY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tiSvN0eo3S
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hq4gn2DqVY
++ cat /tmp/tmp.tiSvN0eo3S
++ rm /tmp/tmp.hq4gn2DqVY /tmp/tmp.tiSvN0eo3S
++ return 0
+ client_pod=pxc-client-59944c5bbf-4nwbf
+ wait_pod pxc-client-59944c5bbf-4nwbf
+ local pod=pxc-client-59944c5bbf-4nwbf
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-4nwbf
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-4nwbf condition met
waiting for pod/pxc-client-59944c5bbf-4nwbf to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.Ww3sIkKqjx/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/compare/select-1.sql /tmp/tmp.Ww3sIkKqjx/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h demand-backup-pxc-2.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-pxc-2.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h demand-backup-pxc-2.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-pxc-2.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5XT5wihfec
+++ mktemp
++ local LAST_ERR=/tmp/tmp.l2LH7vmlnO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.5XT5wihfec
++ cat /tmp/tmp.l2LH7vmlnO
++ rm /tmp/tmp.5XT5wihfec /tmp/tmp.l2LH7vmlnO
++ return 0
+ client_pod=pxc-client-59944c5bbf-4nwbf
+ wait_pod pxc-client-59944c5bbf-4nwbf
+ local pod=pxc-client-59944c5bbf-4nwbf
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-4nwbf
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-4nwbf condition met
waiting for pod/pxc-client-59944c5bbf-4nwbf to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.Ww3sIkKqjx/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/compare/select-1.sql /tmp/tmp.Ww3sIkKqjx/select-1.sql
++ is_keyring_plugin_in_use demand-backup
++ local cluster=demand-backup
++ kubectl_bin exec -it demand-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XS5CSTHvXs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4vExYaxIAw
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it demand-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XS5CSTHvXs
++ cat /tmp/tmp.4vExYaxIAw
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.XS5CSTHvXs /tmp/tmp.4vExYaxIAw
++ return 0
+ '[' '' ']'
+ start_minio demand-backup-ssl
+ deploy_helm demand-backup-20748
+ helm repo add hashicorp https://helm.releases.hashicorp.com
"hashicorp" already exists with the same configuration, skipping
+ helm repo add minio https://charts.min.io/
"minio" already exists with the same configuration, skipping
+ helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "minio" chart repository
...Successfully got an update from the "chaos-mesh" chart repository
...Successfully got an update from the "hashicorp" chart repository
Update Complete. ⎈Happy Helming!⎈
+ local cert_secret=demand-backup-ssl
+ local endpoint=http://minio-service:9000
+ minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G)
+ local minio_args
+ [[ -n demand-backup-ssl ]]
+ endpoint=https://minio-service:9000
+ minio_args+=(--set tls.enabled=true --set tls.certSecret="$cert_secret" --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key)
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=demand-backup-ssl --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=demand-backup-ssl --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio
NAME: minio-service
LAST DEPLOYED: Fri Oct 10 08:53:58 2025
NAMESPACE: demand-backup-20748
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-20748.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-20748 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-20748

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-20748 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-20748 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
+ sleep 30
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ywi7fPET7D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IJGgp4zk0g
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ywi7fPET7D
++ cat /tmp/tmp.IJGgp4zk0g
++ rm /tmp/tmp.ywi7fPET7D /tmp/tmp.IJGgp4zk0g
++ return 0
+ MINIO_POD=minio-service-f46fd9c97-6txj6
+ wait_pod minio-service-f46fd9c97-6txj6
+ local pod=minio-service-f46fd9c97-6txj6
+ local max_retry=480
+ local ns=
++ echo minio-service-f46fd9c97-6txj6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/minio-service-f46fd9c97-6txj6 condition met
waiting for pod/minio-service-f46fd9c97-6txj6 to become Ready.Ok
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url https://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing
++ mktemp
+ local LAST_OUT=/tmp/tmp.HnBuoie4m9
++ mktemp
+ local LAST_ERR=/tmp/tmp.y4xKoGzvg3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url https://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.HnBuoie4m9
/usr/lib/python2.7/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py:768: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.org/en/latest/security.html
  InsecureRequestWarning)
make_bucket: operator-testing
pod "aws-cli" deleted from demand-backup-20748 namespace
+ cat /tmp/tmp.y4xKoGzvg3
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-20748
+ rm /tmp/tmp.HnBuoie4m9 /tmp/tmp.y4xKoGzvg3
+ return 0
+ run_backup demand-backup on-demand-backup-pvc
+ local cluster=demand-backup
+ local backup=on-demand-backup-pvc
+ log 'run pxc-backup/on-demand-backup-pvc'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2025-10-10T08:55:06+0000]' run pxc-backup/on-demand-backup-pvc
[2025-10-10T08:55:06+0000] run pxc-backup/on-demand-backup-pvc
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/conf/on-demand-backup-pvc.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.hVLMIP948R
++ mktemp
+ local LAST_ERR=/tmp/tmp.17F16egW8N
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/conf/on-demand-backup-pvc.yml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/conf/on-demand-backup-pvc.yml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/conf/on-demand-backup-pvc.yml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.hVLMIP948R
+ cat /tmp/tmp.17F16egW8N
error: resource mapping not found for name: "on-demand-backup-pvc" namespace: "" from "/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/demand-backup/conf/on-demand-backup-pvc.yml": no matches for kind "PerconaXtraDBClusterBackup" in version "pxc.percona.com/v1"
ensure CRDs are installed first
+ rm /tmp/tmp.hVLMIP948R /tmp/tmp.17F16egW8N
+ return 1