Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/logs/scheduled-backup-8-0.log
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ main
+ create_infra scheduled-backup-6336
+ local ns=scheduled-backup-6336
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n scheduled-backup-30976 scheduled-backup --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (InternalError): Internal error occurred: failed calling webhook "validationwebhook.pxc.percona.com": failed to call webhook: Post "https://percona-xtradb-cluster-operator.pxc-operator.svc:443/validate-percona-xtradbcluster?timeout=10s": tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "Root CA")
+ :
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.l6hWbV221g
++ mktemp
+ local LAST_ERR=/tmp/tmp.zsm726gNg3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.l6hWbV221g
perconaxtradbcluster.pxc.percona.com "scheduled-backup" deleted from scheduled-backup-30976 namespace
+ cat /tmp/tmp.zsm726gNg3
+ rm /tmp/tmp.l6hWbV221g /tmp/tmp.zsm726gNg3
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.aCDIeTlkf4
++ mktemp
+ local LAST_ERR=/tmp/tmp.kvPfjFb0zf
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.aCDIeTlkf4
+ cat /tmp/tmp.kvPfjFb0zf
error: the server doesn't have a resource type "pxc-backup"
+ rm /tmp/tmp.aCDIeTlkf4 /tmp/tmp.kvPfjFb0zf
+ return 1
+ :
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.vRnaAh2b6v
++ mktemp
+ local LAST_ERR=/tmp/tmp.cUT4amhZEz
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.vRnaAh2b6v
+ cat /tmp/tmp.cUT4amhZEz
error: the server doesn't have a resource type "pxc-restore"
+ rm /tmp/tmp.vRnaAh2b6v /tmp/tmp.cUT4amhZEz
+ return 1
+ :
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ xargs kubectl delete ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.LCafY4ZstD
+ local LAST_OUT=/tmp/tmp.zzasNYOXrK
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.uFSJRXuTKP
+ local exit_status=0
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.GRjh2DaEsQ
+ local exit_status=0
+ awk '{print$1}'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.zzasNYOXrK
+ cat /tmp/tmp.GRjh2DaEsQ
+ rm /tmp/tmp.zzasNYOXrK /tmp/tmp.GRjh2DaEsQ
+ return 0
namespace "scheduled-backup-30976" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.LCafY4ZstD
namespace "pxc-operator" deleted
+ cat /tmp/tmp.uFSJRXuTKP
+ rm /tmp/tmp.LCafY4ZstD /tmp/tmp.uFSJRXuTKP
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.Zmjzplh8yJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.MTAApsX23d
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Zmjzplh8yJ
namespace/pxc-operator created
+ cat /tmp/tmp.MTAApsX23d
+ rm /tmp/tmp.Zmjzplh8yJ /tmp/tmp.MTAApsX23d
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oSr5iLI8ZP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ckgJwSJPOY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.oSr5iLI8ZP
++ cat /tmp/tmp.ckgJwSJPOY
++ rm /tmp/tmp.oSr5iLI8ZP /tmp/tmp.ckgJwSJPOY
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster2 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.B3Nc2a0X0C
++ mktemp
+ local LAST_ERR=/tmp/tmp.47yVNyNslM
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster2 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.B3Nc2a0X0C
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster2" modified.
+ cat /tmp/tmp.47yVNyNslM
+ rm /tmp/tmp.B3Nc2a0X0C /tmp/tmp.47yVNyNslM
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.YF8l7khjOb
++ mktemp
+ local LAST_ERR=/tmp/tmp.AfWDIoHMep
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.YF8l7khjOb
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.AfWDIoHMep
+ rm /tmp/tmp.YF8l7khjOb /tmp/tmp.AfWDIoHMep
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/cw-rbac.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.wiEKQy8vWY
++ mktemp
+ local LAST_ERR=/tmp/tmp.SttNgYlrJW
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.wiEKQy8vWY
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.SttNgYlrJW
+ rm /tmp/tmp.wiEKQy8vWY /tmp/tmp.SttNgYlrJW
+ return 0
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.HzMim66qr2
++ mktemp
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/cw-operator.yaml
+ local LAST_ERR=/tmp/tmp.uRNiBjb5Rd
+ local exit_status=0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7^'
++ seq 0 2
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.HzMim66qr2
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.uRNiBjb5Rd
+ rm /tmp/tmp.HzMim66qr2 /tmp/tmp.uRNiBjb5Rd
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.XHWXKKNve6
++ mktemp
+ local LAST_ERR=/tmp/tmp.GR4aNQybbF
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XHWXKKNve6
pod/percona-xtradb-cluster-operator-859595f865-l96n8 condition met
+ cat /tmp/tmp.GR4aNQybbF
+ rm /tmp/tmp.XHWXKKNve6 /tmp/tmp.GR4aNQybbF
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IV9lmnsRZy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NAuhE98h29
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IV9lmnsRZy
++ cat /tmp/tmp.NAuhE98h29
++ rm /tmp/tmp.IV9lmnsRZy /tmp/tmp.NAuhE98h29
++ return 0
+ wait_pod percona-xtradb-cluster-operator-859595f865-l96n8 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-859595f865-l96n8
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-859595f865-l96n8
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-859595f865-l96n8 condition met
waiting for pod/percona-xtradb-cluster-operator-859595f865-l96n8 to become Ready.Ok
+ sleep 3
+ create_namespace scheduled-backup-6336
+ local namespace=scheduled-backup-6336
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces scheduled-backup-6336'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces scheduled-backup-6336
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace scheduled-backup-6336
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.QDqM7OkRVY
+ local LAST_OUT=/tmp/tmp.RixsL65GHU
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.GRvowrf2dj
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.npsZ37tx3v
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace scheduled-backup-6336
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace scheduled-backup-6336
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QDqM7OkRVY
+ cat /tmp/tmp.npsZ37tx3v
+ rm /tmp/tmp.QDqM7OkRVY /tmp/tmp.npsZ37tx3v
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace scheduled-backup-6336
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.RixsL65GHU
+ cat /tmp/tmp.GRvowrf2dj
Error from server (NotFound): namespaces "scheduled-backup-6336" not found
+ rm /tmp/tmp.RixsL65GHU /tmp/tmp.GRvowrf2dj
+ return 1
+ :
+ wait_for_delete namespace/scheduled-backup-6336
+ local res=namespace/scheduled-backup-6336
+ echo -n 'waiting for namespace/scheduled-backup-6336 to be deleted'
waiting for namespace/scheduled-backup-6336 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "scheduled-backup-6336" not found
+ desc 'create namespace scheduled-backup-6336'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace scheduled-backup-6336
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace scheduled-backup-6336
++ mktemp
+ local LAST_OUT=/tmp/tmp.rR1qaSC689
++ mktemp
+ local LAST_ERR=/tmp/tmp.uy4YesMx2p
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace scheduled-backup-6336
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.rR1qaSC689
namespace/scheduled-backup-6336 created
+ cat /tmp/tmp.uy4YesMx2p
+ rm /tmp/tmp.rR1qaSC689 /tmp/tmp.uy4YesMx2p
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BAaJJqtTlz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IZAgxVLMl5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BAaJJqtTlz
++ cat /tmp/tmp.IZAgxVLMl5
++ rm /tmp/tmp.BAaJJqtTlz /tmp/tmp.IZAgxVLMl5
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster2 --namespace=scheduled-backup-6336
++ mktemp
+ local LAST_OUT=/tmp/tmp.qIDptCkRc0
++ mktemp
+ local LAST_ERR=/tmp/tmp.MdkMGTXSTf
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster2 --namespace=scheduled-backup-6336
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.qIDptCkRc0
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster2" modified.
+ cat /tmp/tmp.MdkMGTXSTf
+ rm /tmp/tmp.qIDptCkRc0 /tmp/tmp.MdkMGTXSTf
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.l39g5N5Ttr
++ mktemp
+ local LAST_ERR=/tmp/tmp.sysP7rg2G3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.l39g5N5Ttr
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.sysP7rg2G3
+ rm /tmp/tmp.l39g5N5Ttr /tmp/tmp.sysP7rg2G3
+ return 0
+ start_minio
+ deploy_helm scheduled-backup-6336
+ helm repo add hashicorp https://helm.releases.hashicorp.com
"hashicorp" already exists with the same configuration, skipping
+ helm repo add minio https://charts.min.io/
"minio" already exists with the same configuration, skipping
+ helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "minio" chart repository
...Successfully got an update from the "chaos-mesh" chart repository
...Successfully got an update from the "percona" chart repository
...Successfully got an update from the "hashicorp" chart repository
Update Complete. ⎈Happy Helming!⎈
+ local cert_secret=
+ local endpoint=http://minio-service:9000
+ minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G)
+ local minio_args
+ [[ -n '' ]]
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio
NAME: minio-service
LAST DEPLOYED: Fri Oct 10 10:02:27 2025
NAMESPACE: scheduled-backup-6336
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.scheduled-backup-6336.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-6336 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-6336

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-6336 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-6336 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
+ sleep 30
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ga0KHnudaP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HyWDdklkdt
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ga0KHnudaP
++ cat /tmp/tmp.HyWDdklkdt
++ rm /tmp/tmp.ga0KHnudaP /tmp/tmp.HyWDdklkdt
++ return 0
+ MINIO_POD=minio-service-55fcc5d75f-fr99z
+ wait_pod minio-service-55fcc5d75f-fr99z
+ local pod=minio-service-55fcc5d75f-fr99z
+ local max_retry=480
+ local ns=
++ echo minio-service-55fcc5d75f-fr99z
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/minio-service-55fcc5d75f-fr99z condition met
waiting for pod/minio-service-55fcc5d75f-fr99z to become Ready.Ok
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing
++ mktemp
+ local LAST_OUT=/tmp/tmp.nfoAPfpxuY
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dq719v0WyZ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nfoAPfpxuY
make_bucket: operator-testing
pod "aws-cli" deleted from scheduled-backup-6336 namespace
+ cat /tmp/tmp.Dq719v0WyZ
+ rm /tmp/tmp.nfoAPfpxuY /tmp/tmp.Dq719v0WyZ
+ return 0
+ cluster=scheduled-backup
+ cat -
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.EUBbp3IrSj
++ mktemp
+ local LAST_ERR=/tmp/tmp.ym3yCo3V3Y
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EUBbp3IrSj
priorityclass.scheduling.k8s.io/high-priority configured
+ cat /tmp/tmp.ym3yCo3V3Y
+ rm /tmp/tmp.EUBbp3IrSj /tmp/tmp.ym3yCo3V3Y
+ return 0
+ spinup_pxc scheduled-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml
+ local cluster=scheduled-backup
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.F1DDqqyJIq
++ mktemp
+ local LAST_ERR=/tmp/tmp.aSkuqdcF4S
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.F1DDqqyJIq
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.aSkuqdcF4S
+ rm /tmp/tmp.F1DDqqyJIq /tmp/tmp.aSkuqdcF4S
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7#'
++ mktemp
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-6336~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ local LAST_OUT=/tmp/tmp.0WvmOecgZ3
++ mktemp
+ local LAST_ERR=/tmp/tmp.pylDyXNAwx
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.0WvmOecgZ3
deployment.apps/pxc-client created
+ cat /tmp/tmp.pylDyXNAwx
+ rm /tmp/tmp.0WvmOecgZ3 /tmp/tmp.pylDyXNAwx
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-6336~
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.8tBpvzZgXS
++ mktemp
+ local LAST_ERR=/tmp/tmp.kaK0mfuZoU
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.8tBpvzZgXS
perconaxtradbcluster.pxc.percona.com/scheduled-backup created
+ cat /tmp/tmp.kaK0mfuZoU
+ rm /tmp/tmp.8tBpvzZgXS /tmp/tmp.kaK0mfuZoU
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy scheduled-backup
++ local target_cluster=scheduled-backup
+++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.x9jASMWvkP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IqxFtEOnhR
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.x9jASMWvkP
+++ cat /tmp/tmp.IqxFtEOnhR
+++ rm /tmp/tmp.x9jASMWvkP /tmp/tmp.IqxFtEOnhR
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.CR94HL0eGv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ggYXEaW6kT
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.CR94HL0eGv
+++ cat /tmp/tmp.ggYXEaW6kT
+++ rm /tmp/tmp.CR94HL0eGv /tmp/tmp.ggYXEaW6kT
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo scheduled-backup-proxysql
++ return
+ local proxy=scheduled-backup-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-6336
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ivsh6D5cU4
++ mktemp
+ local LAST_ERR=/tmp/tmp.FpRkHfoQdc
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-6336
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-6336
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-6336
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.Ivsh6D5cU4
+ cat /tmp/tmp.FpRkHfoQdc
error: no matching resources found
+ rm /tmp/tmp.Ivsh6D5cU4 /tmp/tmp.FpRkHfoQdc
+ return 1
+ true
+ wait_for_running scheduled-backup-proxysql 1
+ local name=scheduled-backup-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod scheduled-backup-proxysql-0 480
+ local pod=scheduled-backup-proxysql-0
+ local max_retry=480
+ local ns=
++ echo scheduled-backup-proxysql-0
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=proxysql
+ set +o xtrace
pod/scheduled-backup-proxysql-0 condition met
waiting for pod/scheduled-backup-proxysql-0 to become Ready.Ok
+ wait_for_running scheduled-backup-pxc 3
+ local name=scheduled-backup-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod scheduled-backup-pxc-0 480
+ local pod=scheduled-backup-pxc-0
+ local max_retry=480
+ local ns=
++ echo scheduled-backup-pxc-0
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/scheduled-backup-pxc-0 condition met
waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod scheduled-backup-pxc-1 480
+ local pod=scheduled-backup-pxc-1
+ local max_retry=480
+ local ns=
++ echo scheduled-backup-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/scheduled-backup-pxc-1 condition met
waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod scheduled-backup-pxc-2 480
+ local pod=scheduled-backup-pxc-2
+ local max_retry=480
+ local ns=
++ echo scheduled-backup-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/scheduled-backup-pxc-2 condition met
waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok
+ sleep 10
++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dTnxcbFrQQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RCiGoKO8Js
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.dTnxcbFrQQ
++ cat /tmp/tmp.RCiGoKO8Js
++ rm /tmp/tmp.dTnxcbFrQQ /tmp/tmp.RCiGoKO8Js
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ojGXi2MJYY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GJaS7fqWDx
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ojGXi2MJYY
++ cat /tmp/tmp.GJaS7fqWDx
++ rm /tmp/tmp.ojGXi2MJYY /tmp/tmp.GJaS7fqWDx
++ return 0
+ client_pod=pxc-client-59944c5bbf-jfm4j
+ wait_pod pxc-client-59944c5bbf-jfm4j
+ local pod=pxc-client-59944c5bbf-jfm4j
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-jfm4j
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-jfm4j condition met
waiting for pod/pxc-client-59944c5bbf-jfm4j to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5d3gN5zviU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dFRhUUDq9M
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.5d3gN5zviU
++ cat /tmp/tmp.dFRhUUDq9M
++ rm /tmp/tmp.5d3gN5zviU /tmp/tmp.dFRhUUDq9M
++ return 0
+ client_pod=pxc-client-59944c5bbf-jfm4j
+ wait_pod pxc-client-59944c5bbf-jfm4j
+ local pod=pxc-client-59944c5bbf-jfm4j
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-jfm4j
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-jfm4j condition met
waiting for pod/pxc-client-59944c5bbf-jfm4j to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q5a5S5kQQl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3Z8LYmJiTY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.q5a5S5kQQl
++ cat /tmp/tmp.3Z8LYmJiTY
++ rm /tmp/tmp.q5a5S5kQQl /tmp/tmp.3Z8LYmJiTY
++ return 0
+ client_pod=pxc-client-59944c5bbf-jfm4j
+ wait_pod pxc-client-59944c5bbf-jfm4j
+ local pod=pxc-client-59944c5bbf-jfm4j
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-59944c5bbf-jfm4j
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-jfm4j condition met
waiting for pod/pxc-client-59944c5bbf-jfm4j to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.vDDOF9UEas/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.vDDOF9UEas/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nT1znWdbb0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ACeqygLXjn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nT1znWdbb0
++ cat /tmp/tmp.ACeqygLXjn
++ rm /tmp/tmp.nT1znWdbb0 /tmp/tmp.ACeqygLXjn
++ return 0
+ client_pod=pxc-client-59944c5bbf-jfm4j
+ wait_pod pxc-client-59944c5bbf-jfm4j
+ local pod=pxc-client-59944c5bbf-jfm4j
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-jfm4j
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-jfm4j condition met
waiting for pod/pxc-client-59944c5bbf-jfm4j to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.vDDOF9UEas/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.vDDOF9UEas/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iVGQL5Xe3R
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RvSZa16rem
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.iVGQL5Xe3R
++ cat /tmp/tmp.RvSZa16rem
++ rm /tmp/tmp.iVGQL5Xe3R /tmp/tmp.RvSZa16rem
++ return 0
+ client_pod=pxc-client-59944c5bbf-jfm4j
+ wait_pod pxc-client-59944c5bbf-jfm4j
+ local pod=pxc-client-59944c5bbf-jfm4j
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-jfm4j
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-jfm4j condition met
waiting for pod/pxc-client-59944c5bbf-jfm4j to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.vDDOF9UEas/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.vDDOF9UEas/select-1.sql
++ is_keyring_plugin_in_use scheduled-backup
++ local cluster=scheduled-backup
++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Wx6l7E1iVT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6JMX1pFTWW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Wx6l7E1iVT
++ cat /tmp/tmp.6JMX1pFTWW
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.Wx6l7E1iVT /tmp/tmp.6JMX1pFTWW
++ return 0
+ '[' '' ']'
+ sleep 20
+ desc 'add backups schedule for pvc storage'
+ set +o xtrace


-----------------------------------------------------------------------------------
add backups schedule for pvc storage
-----------------------------------------------------------------------------------

++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cxCLMy5JXe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iHHNQfGBr0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.cxCLMy5JXe
++ cat /tmp/tmp.iHHNQfGBr0
++ rm /tmp/tmp.cxCLMy5JXe /tmp/tmp.iHHNQfGBr0
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster2 --namespace=scheduled-backup-6336
++ mktemp
+ local LAST_OUT=/tmp/tmp.a9HDbwgS0b
++ mktemp
+ local LAST_ERR=/tmp/tmp.BJd9OX7NvN
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster2 --namespace=scheduled-backup-6336
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.a9HDbwgS0b
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster2" modified.
+ cat /tmp/tmp.BJd9OX7NvN
+ rm /tmp/tmp.a9HDbwgS0b /tmp/tmp.BJd9OX7NvN
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ local LAST_OUT=/tmp/tmp.tryZqjEcQA
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-6336~
++ mktemp
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ local LAST_ERR=/tmp/tmp.kWcZfGzAJU
+ local exit_status=0
++ seq 0 2
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.tryZqjEcQA
+ cat /tmp/tmp.kWcZfGzAJU
error: no objects passed to apply
+ rm /tmp/tmp.tryZqjEcQA /tmp/tmp.kWcZfGzAJU
+ return 1