Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/logs/scheduled-backup-5-7.log Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 + main + create_infra scheduled-backup-22500 + local ns=scheduled-backup-22500 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n scheduled-backup-6885 scheduled-backup --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.Iay8qtEnx8 ++ mktemp + local LAST_ERR=/tmp/tmp.NYscSpCuDk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Iay8qtEnx8 perconaxtradbcluster.pxc.percona.com "scheduled-backup" deleted from scheduled-backup-6885 namespace + cat /tmp/tmp.NYscSpCuDk + rm /tmp/tmp.Iay8qtEnx8 /tmp/tmp.NYscSpCuDk + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.YuDdatEBis ++ mktemp + local LAST_ERR=/tmp/tmp.ko2r513J4L + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YuDdatEBis perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-aws-s3-20264941556-q6fav" deleted from scheduled-backup-6885 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-azure-blob-20264941347-q6fav" deleted from scheduled-backup-6885 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-gcp-cs-20264941143-q6fav" deleted from scheduled-backup-6885 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-minio-2026494555-q6fav" deleted from scheduled-backup-6885 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-pvc-202649416-q6fav" deleted from scheduled-backup-6885 namespace + cat /tmp/tmp.ko2r513J4L + rm /tmp/tmp.YuDdatEBis /tmp/tmp.ko2r513J4L + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.P6g5dyZUT9 ++ mktemp + local LAST_ERR=/tmp/tmp.pidhbjRRJh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.P6g5dyZUT9 perconaxtradbclusterrestore.pxc.percona.com "aws-s3-20264941556-q6fav" deleted from scheduled-backup-6885 namespace perconaxtradbclusterrestore.pxc.percona.com "gcp-cs-20264941143-q6fav" deleted from scheduled-backup-6885 namespace perconaxtradbclusterrestore.pxc.percona.com "minio-2026494555-q6fav" deleted from scheduled-backup-6885 namespace perconaxtradbclusterrestore.pxc.percona.com "pvc-202649416-q6fav" deleted from scheduled-backup-6885 namespace + cat /tmp/tmp.pidhbjRRJh + rm /tmp/tmp.P6g5dyZUT9 /tmp/tmp.pidhbjRRJh + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + awk '{print$1}' + local LAST_OUT=/tmp/tmp.im4aioqTCk ++ mktemp + local LAST_ERR=/tmp/tmp.8k5Ttwhag6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.nbKAdS7JBX ++ mktemp + local LAST_ERR=/tmp/tmp.cITcVJqRlM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nbKAdS7JBX + cat /tmp/tmp.cITcVJqRlM + rm /tmp/tmp.nbKAdS7JBX /tmp/tmp.cITcVJqRlM + return 0 namespace "scheduled-backup-6885" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.im4aioqTCk namespace "pxc-operator" deleted + cat /tmp/tmp.8k5Ttwhag6 + rm /tmp/tmp.im4aioqTCk /tmp/tmp.8k5Ttwhag6 + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.uuvt04QTmZ ++ mktemp + local LAST_ERR=/tmp/tmp.vIbz0JuNMF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uuvt04QTmZ namespace/pxc-operator created + cat /tmp/tmp.vIbz0JuNMF + rm /tmp/tmp.uuvt04QTmZ /tmp/tmp.vIbz0JuNMF + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.ieNOagtJeO +++ mktemp ++ local LAST_ERR=/tmp/tmp.c2mmeBTVLq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ieNOagtJeO ++ cat /tmp/tmp.c2mmeBTVLq ++ rm /tmp/tmp.ieNOagtJeO /tmp/tmp.c2mmeBTVLq ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster7 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.pX1kjoVGKH ++ mktemp + local LAST_ERR=/tmp/tmp.kFZjEgnCqL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster7 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pX1kjoVGKH Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster7" modified. + cat /tmp/tmp.kFZjEgnCqL + rm /tmp/tmp.pX1kjoVGKH /tmp/tmp.kFZjEgnCqL + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.KiqUg87TNH ++ mktemp + local LAST_ERR=/tmp/tmp.EBEfaNPNfL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KiqUg87TNH customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.EBEfaNPNfL + rm /tmp/tmp.KiqUg87TNH /tmp/tmp.EBEfaNPNfL + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/deploy/cw-rbac.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.ritRRj7dia ++ mktemp + local LAST_ERR=/tmp/tmp.ztkr6tMlYe + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ritRRj7dia clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.ztkr6tMlYe + rm /tmp/tmp.ritRRj7dia /tmp/tmp.ztkr6tMlYe + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' ++ mktemp + local LAST_OUT=/tmp/tmp.EGuuRCWFQj ++ mktemp + local LAST_ERR=/tmp/tmp.m9PhRaq9pi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EGuuRCWFQj deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.m9PhRaq9pi + rm /tmp/tmp.EGuuRCWFQj /tmp/tmp.m9PhRaq9pi + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.W8EeQGUbLA ++ mktemp + local LAST_ERR=/tmp/tmp.44i8dvLfUu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.W8EeQGUbLA pod/percona-xtradb-cluster-operator-7d5d85b46b-lvp4d condition met + cat /tmp/tmp.44i8dvLfUu + rm /tmp/tmp.W8EeQGUbLA /tmp/tmp.44i8dvLfUu + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.DpJbx24Ku2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.sC4KyofBgT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DpJbx24Ku2 ++ cat /tmp/tmp.sC4KyofBgT ++ rm /tmp/tmp.DpJbx24Ku2 /tmp/tmp.sC4KyofBgT ++ return 0 + wait_pod percona-xtradb-cluster-operator-7d5d85b46b-lvp4d 480 pxc-operator + local pod=percona-xtradb-cluster-operator-7d5d85b46b-lvp4d + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-7d5d85b46b-lvp4d ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-7d5d85b46b-lvp4d condition met waiting for pod/percona-xtradb-cluster-operator-7d5d85b46b-lvp4d to become Ready.Ok + sleep 3 + create_namespace scheduled-backup-22500 + local namespace=scheduled-backup-22500 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces scheduled-backup-22500' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces scheduled-backup-22500 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace scheduled-backup-22500 ++ mktemp + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.aJgSRRk0qU ++ mktemp + local LAST_OUT=/tmp/tmp.8GLfrh7iXh ++ mktemp + local LAST_ERR=/tmp/tmp.boSRjyHFXf + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.KqOD8IOaal + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-22500 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-22500 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8GLfrh7iXh + cat /tmp/tmp.KqOD8IOaal + rm /tmp/tmp.8GLfrh7iXh /tmp/tmp.KqOD8IOaal + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-22500 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.aJgSRRk0qU + cat /tmp/tmp.boSRjyHFXf Error from server (NotFound): namespaces "scheduled-backup-22500" not found + rm /tmp/tmp.aJgSRRk0qU /tmp/tmp.boSRjyHFXf + return 1 + : + wait_for_delete namespace/scheduled-backup-22500 + local res=namespace/scheduled-backup-22500 + echo -n 'waiting for namespace/scheduled-backup-22500 to be deleted' waiting for namespace/scheduled-backup-22500 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "scheduled-backup-22500" not found + desc 'create namespace scheduled-backup-22500' + set +o xtrace ----------------------------------------------------------------------------------- create namespace scheduled-backup-22500 ----------------------------------------------------------------------------------- + kubectl_bin create namespace scheduled-backup-22500 ++ mktemp + local LAST_OUT=/tmp/tmp.7XokTRq7DP ++ mktemp + local LAST_ERR=/tmp/tmp.jyrzatT5nx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace scheduled-backup-22500 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7XokTRq7DP namespace/scheduled-backup-22500 created + cat /tmp/tmp.jyrzatT5nx + rm /tmp/tmp.7XokTRq7DP /tmp/tmp.jyrzatT5nx + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.PNCdolffo4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.lzIMA96a27 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PNCdolffo4 ++ cat /tmp/tmp.lzIMA96a27 ++ rm /tmp/tmp.PNCdolffo4 /tmp/tmp.lzIMA96a27 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster7 --namespace=scheduled-backup-22500 ++ mktemp + local LAST_OUT=/tmp/tmp.BQCbQNl216 ++ mktemp + local LAST_ERR=/tmp/tmp.hV4PZJ0HKZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster7 --namespace=scheduled-backup-22500 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BQCbQNl216 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster7" modified. + cat /tmp/tmp.hV4PZJ0HKZ + rm /tmp/tmp.BQCbQNl216 /tmp/tmp.hV4PZJ0HKZ + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xFhe2OVUqW ++ mktemp + local LAST_ERR=/tmp/tmp.xmbn9lreJ9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xFhe2OVUqW secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.xmbn9lreJ9 + rm /tmp/tmp.xFhe2OVUqW /tmp/tmp.xmbn9lreJ9 + return 0 + start_minio + deploy_helm scheduled-backup-22500 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "hashicorp" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Thu Apr 9 04:48:50 2026 NAMESPACE: scheduled-backup-22500 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.scheduled-backup-22500.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-22500 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-22500 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-22500 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-22500 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yKHtxEMeXi +++ mktemp ++ local LAST_ERR=/tmp/tmp.xhOSpVtPqx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yKHtxEMeXi ++ cat /tmp/tmp.xhOSpVtPqx ++ rm /tmp/tmp.yKHtxEMeXi /tmp/tmp.xhOSpVtPqx ++ return 0 + MINIO_POD=minio-service-5fd5489bdc-4lfvk + wait_pod minio-service-5fd5489bdc-4lfvk + local pod=minio-service-5fd5489bdc-4lfvk + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo minio-service-5fd5489bdc-4lfvk + local container= + set +o xtrace pod/minio-service-5fd5489bdc-4lfvk condition met waiting for pod/minio-service-5fd5489bdc-4lfvk to become Ready.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.9hCW935p1F ++ mktemp + local LAST_ERR=/tmp/tmp.c8I290bw6N + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9hCW935p1F make_bucket: operator-testing pod "aws-cli" deleted from scheduled-backup-22500 namespace + cat /tmp/tmp.c8I290bw6N + rm /tmp/tmp.9hCW935p1F /tmp/tmp.c8I290bw6N + return 0 + cluster=scheduled-backup + cat - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.TtBBY9Qvgg ++ mktemp + local LAST_ERR=/tmp/tmp.6jzMnMy7kT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TtBBY9Qvgg priorityclass.scheduling.k8s.io/high-priority configured + cat /tmp/tmp.6jzMnMy7kT + rm /tmp/tmp.TtBBY9Qvgg /tmp/tmp.6jzMnMy7kT + return 0 + spinup_pxc scheduled-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local cluster=scheduled-backup + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.5dFYzjz7wx ++ mktemp + local LAST_ERR=/tmp/tmp.4gaOWrBGvh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5dFYzjz7wx secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.4gaOWrBGvh + rm /tmp/tmp.5dFYzjz7wx /tmp/tmp.4gaOWrBGvh + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml + kubectl_bin apply -f - + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.NE3u8V9BJ9 + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + local LAST_ERR=/tmp/tmp.jwBR5XTJis + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NE3u8V9BJ9 deployment.apps/pxc-client created + cat /tmp/tmp.jwBR5XTJis + rm /tmp/tmp.NE3u8V9BJ9 /tmp/tmp.jwBR5XTJis + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local pvc_name= + kubectl_bin apply -f - ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local LAST_OUT=/tmp/tmp.GBWvCgdjKC + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + local LAST_ERR=/tmp/tmp.ZZqOecxoZB + local exit_status=0 + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GBWvCgdjKC perconaxtradbcluster.pxc.percona.com/scheduled-backup created + cat /tmp/tmp.ZZqOecxoZB + rm /tmp/tmp.GBWvCgdjKC /tmp/tmp.ZZqOecxoZB + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy scheduled-backup ++ local target_cluster=scheduled-backup +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ZjInk9987H ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MesT0bKOVi +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.ZjInk9987H +++ cat /tmp/tmp.MesT0bKOVi +++ rm /tmp/tmp.ZjInk9987H /tmp/tmp.MesT0bKOVi +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7szIILTYco ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OVYLtnydrf +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.7szIILTYco +++ cat /tmp/tmp.OVYLtnydrf +++ rm /tmp/tmp.7szIILTYco /tmp/tmp.OVYLtnydrf +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scheduled-backup-proxysql ++ return + local proxy=scheduled-backup-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-22500 ++ mktemp + local LAST_OUT=/tmp/tmp.MHJVRLLKHm ++ mktemp + local LAST_ERR=/tmp/tmp.jzeqZAGmcN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-22500 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-22500 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-22500 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.MHJVRLLKHm + cat /tmp/tmp.jzeqZAGmcN error: no matching resources found + rm /tmp/tmp.MHJVRLLKHm /tmp/tmp.jzeqZAGmcN + return 1 + true + wait_for_running scheduled-backup-proxysql 1 + local name=scheduled-backup-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-proxysql-0 480 + local pod=scheduled-backup-proxysql-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/scheduled-backup-proxysql-0 condition met waiting for pod/scheduled-backup-proxysql-0 to become Ready.Ok + wait_for_running scheduled-backup-pxc 3 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 480 + local pod=scheduled-backup-pxc-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 480 + local pod=scheduled-backup-pxc-1 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 480 + local pod=scheduled-backup-pxc-2 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Zb4JdsBhfb +++ mktemp ++ local LAST_ERR=/tmp/tmp.NTl39DOh8t ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Zb4JdsBhfb ++ cat /tmp/tmp.NTl39DOh8t ++ rm /tmp/tmp.Zb4JdsBhfb /tmp/tmp.NTl39DOh8t ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] ++ kubectl exec scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ grep -E -o 'early-plugin-load=keyring_\w+.so' ++ return 1 + [[ -n '' ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LJMuvzpVG6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.J8galQGhAs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LJMuvzpVG6 ++ cat /tmp/tmp.J8galQGhAs ++ rm /tmp/tmp.LJMuvzpVG6 /tmp/tmp.J8galQGhAs ++ return 0 + client_pod=pxc-client-97cb9c68b-k2wtj + wait_pod pxc-client-97cb9c68b-k2wtj + local pod=pxc-client-97cb9c68b-k2wtj + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-97cb9c68b-k2wtj + local container= + set +o xtrace pod/pxc-client-97cb9c68b-k2wtj condition met waiting for pod/pxc-client-97cb9c68b-k2wtj to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gPW9KnROzI +++ mktemp ++ local LAST_ERR=/tmp/tmp.rWxcRsU5YD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gPW9KnROzI ++ cat /tmp/tmp.rWxcRsU5YD ++ rm /tmp/tmp.gPW9KnROzI /tmp/tmp.rWxcRsU5YD ++ return 0 + client_pod=pxc-client-97cb9c68b-k2wtj + wait_pod pxc-client-97cb9c68b-k2wtj + local pod=pxc-client-97cb9c68b-k2wtj + local max_retry=480 + local ns= ++ echo pxc-client-97cb9c68b-k2wtj ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-97cb9c68b-k2wtj condition met waiting for pod/pxc-client-97cb9c68b-k2wtj to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AdoUglMtVS +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ko1Qh3NHAb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AdoUglMtVS ++ cat /tmp/tmp.Ko1Qh3NHAb ++ rm /tmp/tmp.AdoUglMtVS /tmp/tmp.Ko1Qh3NHAb ++ return 0 + client_pod=pxc-client-97cb9c68b-k2wtj + wait_pod pxc-client-97cb9c68b-k2wtj + local pod=pxc-client-97cb9c68b-k2wtj + local max_retry=480 + local ns= ++ echo pxc-client-97cb9c68b-k2wtj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-97cb9c68b-k2wtj condition met waiting for pod/pxc-client-97cb9c68b-k2wtj to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.ipBQjx3O3C/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.ipBQjx3O3C/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.ipBQjx3O3C/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mCiAetfFbE +++ mktemp ++ local LAST_ERR=/tmp/tmp.CBC0RmyevK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mCiAetfFbE ++ cat /tmp/tmp.CBC0RmyevK ++ rm /tmp/tmp.mCiAetfFbE /tmp/tmp.CBC0RmyevK ++ return 0 + client_pod=pxc-client-97cb9c68b-k2wtj + wait_pod pxc-client-97cb9c68b-k2wtj + local pod=pxc-client-97cb9c68b-k2wtj + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-97cb9c68b-k2wtj ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-97cb9c68b-k2wtj condition met waiting for pod/pxc-client-97cb9c68b-k2wtj to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.ipBQjx3O3C/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.ipBQjx3O3C/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.ipBQjx3O3C/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cwuez1t082 +++ mktemp ++ local LAST_ERR=/tmp/tmp.64XxJbg1D2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cwuez1t082 ++ cat /tmp/tmp.64XxJbg1D2 ++ rm /tmp/tmp.cwuez1t082 /tmp/tmp.64XxJbg1D2 ++ return 0 + client_pod=pxc-client-97cb9c68b-k2wtj + wait_pod pxc-client-97cb9c68b-k2wtj + local pod=pxc-client-97cb9c68b-k2wtj + local max_retry=480 + local ns= ++ echo pxc-client-97cb9c68b-k2wtj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-97cb9c68b-k2wtj condition met waiting for pod/pxc-client-97cb9c68b-k2wtj to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.ipBQjx3O3C/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.ipBQjx3O3C/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.ipBQjx3O3C/select-1.sql + is_keyring_plugin_in_use scheduled-backup + local cluster=scheduled-backup + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + kubectl exec scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + grep -E -o 'early-plugin-load=keyring_\w+.so' + return 1 + sleep 20 + desc 'add backups schedule for pvc storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for pvc storage ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.q2B7rA0WrU +++ mktemp ++ local LAST_ERR=/tmp/tmp.CRAKbjUNqV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.q2B7rA0WrU ++ cat /tmp/tmp.CRAKbjUNqV ++ rm /tmp/tmp.q2B7rA0WrU /tmp/tmp.CRAKbjUNqV ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster7 --namespace=scheduled-backup-22500 ++ mktemp + local LAST_OUT=/tmp/tmp.QaMx0HCBIv ++ mktemp + local LAST_ERR=/tmp/tmp.9qBhRFl08b + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster7 --namespace=scheduled-backup-22500 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QaMx0HCBIv Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster7" modified. + cat /tmp/tmp.9qBhRFl08b + rm /tmp/tmp.QaMx0HCBIv /tmp/tmp.9qBhRFl08b + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local LAST_OUT=/tmp/tmp.BBJ0BtwWdH ++ mktemp + local LAST_ERR=/tmp/tmp.wtTsZCKXaq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BBJ0BtwWdH perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.wtTsZCKXaq + rm /tmp/tmp.BBJ0BtwWdH /tmp/tmp.wtTsZCKXaq + return 0 + label_node ++ kubectl_bin get nodes --no-headers=true ++ grep -v master +++ mktemp ++ head -n1 ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.fi0ASEPdPR +++ mktemp ++ local LAST_ERR=/tmp/tmp.6OlMdNVic6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get nodes --no-headers=true ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fi0ASEPdPR ++ cat /tmp/tmp.6OlMdNVic6 ++ rm /tmp/tmp.fi0ASEPdPR /tmp/tmp.6OlMdNVic6 ++ return 0 + LABELED_NODE=gke-jen-pxc-2433-150d061-default-pool-40f1c7c4-34n3 + kubectl_bin label nodes gke-jen-pxc-2433-150d061-default-pool-40f1c7c4-34n3 backupWorker=True --overwrite ++ mktemp + local LAST_OUT=/tmp/tmp.kpvfshl0R2 ++ mktemp + local LAST_ERR=/tmp/tmp.zaJA8ZrY6C + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label nodes gke-jen-pxc-2433-150d061-default-pool-40f1c7c4-34n3 backupWorker=True --overwrite + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kpvfshl0R2 node/gke-jen-pxc-2433-150d061-default-pool-40f1c7c4-34n3 not labeled + cat /tmp/tmp.zaJA8ZrY6C + rm /tmp/tmp.kpvfshl0R2 /tmp/tmp.zaJA8ZrY6C + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + local LAST_OUT=/tmp/tmp.0DBOCc5il4 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ ++ mktemp + local LAST_ERR=/tmp/tmp.v5LLGt3sEN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0DBOCc5il4 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.v5LLGt3sEN + rm /tmp/tmp.0DBOCc5il4 /tmp/tmp.v5LLGt3sEN + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.lqVGyCwU22 +++ mktemp ++ local LAST_ERR=/tmp/tmp.cHJYwTUKij ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lqVGyCwU22 ++ cat /tmp/tmp.cHJYwTUKij ++ rm /tmp/tmp.lqVGyCwU22 /tmp/tmp.cHJYwTUKij ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.u3ZMbcDPxB +++ mktemp ++ local LAST_ERR=/tmp/tmp.3FDrmPZ20Z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u3ZMbcDPxB ++ cat /tmp/tmp.3FDrmPZ20Z ++ rm /tmp/tmp.u3ZMbcDPxB /tmp/tmp.3FDrmPZ20Z ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.L5G5tz7VLp ++ mktemp + local LAST_ERR=/tmp/tmp.sqZQz44EPG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.L5G5tz7VLp NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-2026494562-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2026494562-q6fav-20260409045602-ea697605 Running 27s + cat /tmp/tmp.sqZQz44EPG + rm /tmp/tmp.L5G5tz7VLp /tmp/tmp.sqZQz44EPG + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.kzueZROCjc ++ mktemp + local LAST_ERR=/tmp/tmp.kxEyC0pziR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kzueZROCjc NAME READY STATUS RESTARTS AGE minio-service-5fd5489bdc-4lfvk 1/1 Running 0 7m37s pxc-client-97cb9c68b-k2wtj 2/2 Running 0 6m27s scheduled-backup-proxysql-0 3/3 Running 0 6m24s scheduled-backup-proxysql-1 3/3 Running 0 6m3s scheduled-backup-pxc-0 1/1 Running 0 6m24s scheduled-backup-pxc-1 1/1 Running 0 5m3s scheduled-backup-pxc-2 1/1 Running 0 4m13s xb-cron-scheduled-backup-pvc-2026494562-q6fav-6clst 1/1 Running 0 28s + cat /tmp/tmp.kxEyC0pziR + rm /tmp/tmp.kzueZROCjc /tmp/tmp.kxEyC0pziR + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eO3vtTtRxQ ++ wc -l ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ local LAST_ERR=/tmp/tmp.GvoXpsvjWq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eO3vtTtRxQ ++ cat /tmp/tmp.GvoXpsvjWq ++ rm /tmp/tmp.eO3vtTtRxQ /tmp/tmp.GvoXpsvjWq ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.1bZRimTjHK +++ mktemp ++ local LAST_ERR=/tmp/tmp.H8bmxv0ljk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1bZRimTjHK ++ cat /tmp/tmp.H8bmxv0ljk ++ rm /tmp/tmp.1bZRimTjHK /tmp/tmp.H8bmxv0ljk ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local pvc_name= + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + local LAST_OUT=/tmp/tmp.jVV5GNb0GV ++ mktemp + local LAST_ERR=/tmp/tmp.YQRxSRzgwH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jVV5GNb0GV perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.YQRxSRzgwH + rm /tmp/tmp.jVV5GNb0GV /tmp/tmp.YQRxSRzgwH + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.D7vFhktd0Q + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_ERR=/tmp/tmp.c4uGiHgaOg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.D7vFhktd0Q perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.c4uGiHgaOg + rm /tmp/tmp.D7vFhktd0Q /tmp/tmp.c4uGiHgaOg + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VJvLi1SV2h +++ mktemp ++ local LAST_ERR=/tmp/tmp.0wWREndXXo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VJvLi1SV2h ++ cat /tmp/tmp.0wWREndXXo ++ rm /tmp/tmp.VJvLi1SV2h /tmp/tmp.0wWREndXXo ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KBFAps1MQU +++ mktemp ++ local LAST_ERR=/tmp/tmp.M2jj2gwrNe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KBFAps1MQU ++ cat /tmp/tmp.M2jj2gwrNe ++ rm /tmp/tmp.KBFAps1MQU /tmp/tmp.M2jj2gwrNe ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.7FJQ3zyVjS ++ mktemp + local LAST_ERR=/tmp/tmp.AQwoY0CnYY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7FJQ3zyVjS NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20264945757-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2026-04-09-04:57:57-full Running 24s cron-scheduled-backup-pvc-2026494562-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2026494562-q6fav-20260409045602-ea697605 Succeeded 107s 2m19s + cat /tmp/tmp.AQwoY0CnYY + rm /tmp/tmp.7FJQ3zyVjS /tmp/tmp.AQwoY0CnYY + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.1gz3Sv3Oei ++ mktemp + local LAST_ERR=/tmp/tmp.YWna4CzQGw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1gz3Sv3Oei NAME READY STATUS RESTARTS AGE minio-service-5fd5489bdc-4lfvk 1/1 Running 0 9m29s pxc-client-97cb9c68b-k2wtj 2/2 Running 0 8m19s scheduled-backup-proxysql-0 3/3 Running 0 8m16s scheduled-backup-proxysql-1 3/3 Running 0 7m55s scheduled-backup-pxc-0 1/1 Running 0 8m16s scheduled-backup-pxc-1 1/1 Running 0 6m55s scheduled-backup-pxc-2 1/1 Running 0 6m5s xb-cron-scheduled-backup-aws-s3-20264945757-q6fav-8s6nr 1/1 Running 0 25s xb-cron-scheduled-backup-pvc-2026494562-q6fav-6clst 0/1 Completed 0 2m20s + cat /tmp/tmp.YWna4CzQGw + rm /tmp/tmp.1gz3Sv3Oei /tmp/tmp.YWna4CzQGw + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.TehJW4j3CV +++ mktemp ++ local LAST_ERR=/tmp/tmp.zYR4lZTby5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TehJW4j3CV ++ cat /tmp/tmp.zYR4lZTby5 ++ rm /tmp/tmp.TehJW4j3CV /tmp/tmp.zYR4lZTby5 ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.SKUVOKeSBv +++ mktemp ++ local LAST_ERR=/tmp/tmp.7AXVgE3mZM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SKUVOKeSBv ++ cat /tmp/tmp.7AXVgE3mZM ++ rm /tmp/tmp.SKUVOKeSBv /tmp/tmp.7AXVgE3mZM ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for minio storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for minio storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + kubectl_bin apply -f - + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.oWnilO9Idd + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.lZZD9KVBDQ + local exit_status=0 + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ seq 0 2 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oWnilO9Idd perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.lZZD9KVBDQ + rm /tmp/tmp.oWnilO9Idd /tmp/tmp.lZZD9KVBDQ + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + kubectl_bin apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local LAST_OUT=/tmp/tmp.6w4HwRBSwR + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + local LAST_ERR=/tmp/tmp.Mcr0SvCoSt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6w4HwRBSwR perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.Mcr0SvCoSt + rm /tmp/tmp.6w4HwRBSwR /tmp/tmp.Mcr0SvCoSt + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.LqxrhbUkfR +++ mktemp ++ local LAST_ERR=/tmp/tmp.cshlTG2zZw ++ local exit_status=0 ++ grep -vE ':Succeeded|:Failed' +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LqxrhbUkfR ++ cat /tmp/tmp.cshlTG2zZw ++ rm /tmp/tmp.LqxrhbUkfR /tmp/tmp.cshlTG2zZw ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ grep :Failed ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dbXwSNo8pE +++ mktemp ++ local LAST_ERR=/tmp/tmp.q8k0KBRTYs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dbXwSNo8pE ++ cat /tmp/tmp.q8k0KBRTYs ++ rm /tmp/tmp.dbXwSNo8pE /tmp/tmp.q8k0KBRTYs ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-1 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.PzXxl95P3F ++ mktemp + local LAST_ERR=/tmp/tmp.e8L1zbEiyJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PzXxl95P3F NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20264945757-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2026-04-09-04:57:57-full Succeeded 110s 2m31s cron-scheduled-backup-minio-20264945948-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2026-04-09-04:59:48-full Succeeded 10s 40s cron-scheduled-backup-pvc-2026494562-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2026494562-q6fav-20260409045602-ea697605 Succeeded 3m54s 4m26s + cat /tmp/tmp.e8L1zbEiyJ + rm /tmp/tmp.PzXxl95P3F /tmp/tmp.e8L1zbEiyJ + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.lRvzoQAeLm ++ mktemp + local LAST_ERR=/tmp/tmp.SRm5gQCmcc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lRvzoQAeLm NAME READY STATUS RESTARTS AGE minio-service-5fd5489bdc-4lfvk 1/1 Running 0 11m pxc-client-97cb9c68b-k2wtj 2/2 Running 0 10m scheduled-backup-proxysql-0 3/3 Running 0 10m scheduled-backup-proxysql-1 3/3 Running 0 10m scheduled-backup-pxc-0 1/1 Running 0 10m scheduled-backup-pxc-1 1/1 Running 0 9m2s scheduled-backup-pxc-2 1/1 Running 0 8m12s xb-cron-scheduled-backup-aws-s3-20264945757-q6fav-8s6nr 0/1 Completed 0 2m32s xb-cron-scheduled-backup-minio-20264945948-q6fav-28ffs 0/1 Completed 0 41s xb-cron-scheduled-backup-pvc-2026494562-q6fav-6clst 0/1 Completed 0 4m27s + cat /tmp/tmp.SRm5gQCmcc + rm /tmp/tmp.lRvzoQAeLm /tmp/tmp.SRm5gQCmcc + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Yj7yIwM5Gd +++ mktemp ++ local LAST_ERR=/tmp/tmp.NTJGhA4G6p ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Yj7yIwM5Gd ++ cat /tmp/tmp.NTJGhA4G6p ++ rm /tmp/tmp.Yj7yIwM5Gd /tmp/tmp.NTJGhA4G6p ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.V0D1A3RAGF +++ mktemp ++ local LAST_ERR=/tmp/tmp.TkRWG8P5s5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.V0D1A3RAGF ++ cat /tmp/tmp.TkRWG8P5s5 ++ rm /tmp/tmp.V0D1A3RAGF /tmp/tmp.TkRWG8P5s5 ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for gcs storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for gcs storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.5rrHUoSGZa ++ mktemp + local LAST_ERR=/tmp/tmp.7AN8QeLsS3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5rrHUoSGZa perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.7AN8QeLsS3 + rm /tmp/tmp.5rrHUoSGZa /tmp/tmp.7AN8QeLsS3 + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.1CIjQ4v5x5 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + local LAST_ERR=/tmp/tmp.nMouAW3daP + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1CIjQ4v5x5 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.nMouAW3daP + rm /tmp/tmp.1CIjQ4v5x5 /tmp/tmp.nMouAW3daP + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iPtDt7A0rY +++ mktemp ++ local LAST_ERR=/tmp/tmp.BLTTp0tVcX ++ local exit_status=0 +++ seq 0 2 ++ wc -l ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iPtDt7A0rY ++ cat /tmp/tmp.BLTTp0tVcX ++ rm /tmp/tmp.iPtDt7A0rY /tmp/tmp.BLTTp0tVcX ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3IRrSBYtmW +++ mktemp ++ local LAST_ERR=/tmp/tmp.beieor8Ej2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3IRrSBYtmW ++ cat /tmp/tmp.beieor8Ej2 ++ rm /tmp/tmp.3IRrSBYtmW /tmp/tmp.beieor8Ej2 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-0 + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.jkZXwdVJLd ++ mktemp + local LAST_ERR=/tmp/tmp.0PE64YSHDj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jkZXwdVJLd NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20264945757-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2026-04-09-04:57:57-full Succeeded 3m44s 4m25s cron-scheduled-backup-gcp-cs-2026495157-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2026-04-09-05:01:57-full Running 25s cron-scheduled-backup-minio-20264945948-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2026-04-09-04:59:48-full Succeeded 2m4s 2m34s cron-scheduled-backup-pvc-2026494562-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2026494562-q6fav-20260409045602-ea697605 Succeeded 5m48s 6m20s + cat /tmp/tmp.0PE64YSHDj + rm /tmp/tmp.jkZXwdVJLd /tmp/tmp.0PE64YSHDj + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.afrWy4e4E0 ++ mktemp + local LAST_ERR=/tmp/tmp.SHHvA8amjX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.afrWy4e4E0 NAME READY STATUS RESTARTS AGE minio-service-5fd5489bdc-4lfvk 1/1 Running 0 13m pxc-client-97cb9c68b-k2wtj 2/2 Running 0 12m scheduled-backup-proxysql-0 3/3 Running 0 12m scheduled-backup-proxysql-1 3/3 Running 0 11m scheduled-backup-pxc-0 1/1 Running 0 12m scheduled-backup-pxc-1 1/1 Running 0 10m scheduled-backup-pxc-2 1/1 Running 0 10m xb-cron-scheduled-backup-aws-s3-20264945757-q6fav-8s6nr 0/1 Completed 0 4m26s xb-cron-scheduled-backup-gcp-cs-2026495157-q6fav-mkmf6 1/1 Running 0 26s xb-cron-scheduled-backup-minio-20264945948-q6fav-28ffs 0/1 Completed 0 2m35s xb-cron-scheduled-backup-pvc-2026494562-q6fav-6clst 0/1 Completed 0 6m21s + cat /tmp/tmp.SHHvA8amjX + rm /tmp/tmp.afrWy4e4E0 /tmp/tmp.SHHvA8amjX + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.C1MjxhUamn +++ mktemp ++ local LAST_ERR=/tmp/tmp.Y1TQFLTGMR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.C1MjxhUamn ++ cat /tmp/tmp.Y1TQFLTGMR ++ rm /tmp/tmp.C1MjxhUamn /tmp/tmp.Y1TQFLTGMR ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.YdFfD8obGK +++ mktemp ++ local LAST_ERR=/tmp/tmp.JoO4vJ6zQb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YdFfD8obGK ++ cat /tmp/tmp.JoO4vJ6zQb ++ rm /tmp/tmp.YdFfD8obGK /tmp/tmp.JoO4vJ6zQb ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for azure storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for azure storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + kubectl_bin apply -f - + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.sHgUSkgRu0 ++ mktemp + local LAST_ERR=/tmp/tmp.7FQNJaeTPi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sHgUSkgRu0 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.7FQNJaeTPi + rm /tmp/tmp.sHgUSkgRu0 /tmp/tmp.7FQNJaeTPi + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + local LAST_OUT=/tmp/tmp.eNyCyt081M + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.kBNBBX5PjJ + local exit_status=0 + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eNyCyt081M perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.kBNBBX5PjJ + rm /tmp/tmp.eNyCyt081M /tmp/tmp.kBNBBX5PjJ + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.pkm75zQgGl +++ mktemp ++ local LAST_ERR=/tmp/tmp.rBbCmOOSlE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pkm75zQgGl ++ cat /tmp/tmp.rBbCmOOSlE ++ rm /tmp/tmp.pkm75zQgGl /tmp/tmp.rBbCmOOSlE ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.zmlG4z6Phh +++ mktemp ++ local LAST_ERR=/tmp/tmp.Iz1blhyWus ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zmlG4z6Phh ++ cat /tmp/tmp.Iz1blhyWus ++ rm /tmp/tmp.zmlG4z6Phh /tmp/tmp.Iz1blhyWus ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.ezbnQqUEUc ++ mktemp + local LAST_ERR=/tmp/tmp.As1RwfIhPH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ezbnQqUEUc NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20264945757-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2026-04-09-04:57:57-full Succeeded 5m30s 6m11s cron-scheduled-backup-azure-blob-2026495348-q6fav scheduled-backup azure-blob azure://operator-testing/scheduled-backup-2026-04-09-05:03:48-full Running 20s cron-scheduled-backup-gcp-cs-2026495157-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2026-04-09-05:01:57-full Succeeded 95s 2m11s cron-scheduled-backup-minio-20264945948-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2026-04-09-04:59:48-full Succeeded 3m50s 4m20s cron-scheduled-backup-pvc-2026494562-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2026494562-q6fav-20260409045602-ea697605 Succeeded 7m34s 8m6s + cat /tmp/tmp.As1RwfIhPH + rm /tmp/tmp.ezbnQqUEUc /tmp/tmp.As1RwfIhPH + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.0ONS9Q3TAi ++ mktemp + local LAST_ERR=/tmp/tmp.J5a0Sept7J + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0ONS9Q3TAi NAME READY STATUS RESTARTS AGE minio-service-5fd5489bdc-4lfvk 1/1 Running 0 15m pxc-client-97cb9c68b-k2wtj 2/2 Running 0 14m scheduled-backup-proxysql-0 3/3 Running 0 14m scheduled-backup-proxysql-1 3/3 Running 0 13m scheduled-backup-pxc-0 1/1 Running 0 14m scheduled-backup-pxc-1 1/1 Running 0 12m scheduled-backup-pxc-2 1/1 Running 0 11m xb-cron-scheduled-backup-aws-s3-20264945757-q6fav-8s6nr 0/1 Completed 0 6m11s xb-cron-scheduled-backup-azure-blob-2026495348-q6fav-jhgg4 1/1 Running 0 20s xb-cron-scheduled-backup-gcp-cs-2026495157-q6fav-mkmf6 0/1 Completed 0 2m11s xb-cron-scheduled-backup-minio-20264945948-q6fav-28ffs 0/1 Completed 0 4m20s xb-cron-scheduled-backup-pvc-2026494562-q6fav-6clst 0/1 Completed 0 8m6s + cat /tmp/tmp.J5a0Sept7J + rm /tmp/tmp.0ONS9Q3TAi /tmp/tmp.J5a0Sept7J + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.NyURrMwA9X +++ mktemp ++ local LAST_ERR=/tmp/tmp.OyTFKabTz6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NyURrMwA9X ++ cat /tmp/tmp.OyTFKabTz6 ++ rm /tmp/tmp.NyURrMwA9X /tmp/tmp.OyTFKabTz6 ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep :Failed ++ local LAST_OUT=/tmp/tmp.s0lJ5q9hJV +++ mktemp ++ local LAST_ERR=/tmp/tmp.GW2li7ltMy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.s0lJ5q9hJV ++ cat /tmp/tmp.GW2li7ltMy ++ rm /tmp/tmp.s0lJ5q9hJV /tmp/tmp.GW2li7ltMy ++ return 0 + [[ 0 -gt 0 ]] ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ head -n1 ++ grep pvc ++ cut -d: -f1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ce0hlSzFOV +++ mktemp ++ local LAST_ERR=/tmp/tmp.fW9ogUovtD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ce0hlSzFOV ++ cat /tmp/tmp.fW9ogUovtD ++ rm /tmp/tmp.Ce0hlSzFOV /tmp/tmp.fW9ogUovtD ++ return 0 + FIRST_PVC_BACKUP=cron-scheduled-backup-pvc-2026494562-q6fav ++ kubectl_bin get jobs ++ awk '{print $1}' +++ mktemp ++ grep cron-scheduled-backup-pvc-2026494562-q6fav ++ local LAST_OUT=/tmp/tmp.vDrmASUH3G +++ mktemp ++ local LAST_ERR=/tmp/tmp.F6AL8ck6P6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vDrmASUH3G ++ cat /tmp/tmp.F6AL8ck6P6 ++ rm /tmp/tmp.vDrmASUH3G /tmp/tmp.F6AL8ck6P6 ++ return 0 + JOB_PVC_BACKUP=xb-cron-scheduled-backup-pvc-2026494562-q6fav ++ kubectl_bin get pods ++ awk '{print $1}' +++ mktemp ++ grep xb-cron-scheduled-backup-pvc-2026494562 ++ local LAST_OUT=/tmp/tmp.i7r5xIx3CB +++ mktemp ++ local LAST_ERR=/tmp/tmp.q345LFEiCI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.i7r5xIx3CB ++ cat /tmp/tmp.q345LFEiCI ++ rm /tmp/tmp.i7r5xIx3CB /tmp/tmp.q345LFEiCI ++ return 0 + POD_PVC_BACKUP=xb-cron-scheduled-backup-pvc-2026494562-q6fav-6clst ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ grep minio +++ mktemp ++ head -n1 ++ cut -d: -f1 ++ local LAST_OUT=/tmp/tmp.2NfbplBcqL +++ mktemp ++ local LAST_ERR=/tmp/tmp.BIpGI1y4v2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2NfbplBcqL ++ cat /tmp/tmp.BIpGI1y4v2 ++ rm /tmp/tmp.2NfbplBcqL /tmp/tmp.BIpGI1y4v2 ++ return 0 + FIRST_MINIO_BACKUP=cron-scheduled-backup-minio-20264945948-q6fav ++ grep cron-scheduled-backup-minio-20264945948-q6fav ++ awk '{print $1}' ++ kubectl_bin get jobs +++ mktemp ++ local LAST_OUT=/tmp/tmp.7sHhxoe9o0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.BEUkqeB419 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7sHhxoe9o0 ++ cat /tmp/tmp.BEUkqeB419 ++ rm /tmp/tmp.7sHhxoe9o0 /tmp/tmp.BEUkqeB419 ++ return 0 + JOB_MINIO_BACKUP=xb-cron-scheduled-backup-minio-20264945948-q6fav ++ kubectl_bin get pods ++ awk '{print $1}' ++ grep xb-cron-scheduled-backup-minio-20264945948 +++ mktemp ++ local LAST_OUT=/tmp/tmp.e8CFkVPFVT +++ mktemp ++ local LAST_ERR=/tmp/tmp.A6qQlxqvLt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.e8CFkVPFVT ++ cat /tmp/tmp.A6qQlxqvLt ++ rm /tmp/tmp.e8CFkVPFVT /tmp/tmp.A6qQlxqvLt ++ return 0 + POD_MINIO_BACKUP=xb-cron-scheduled-backup-minio-20264945948-q6fav-28ffs + '[' -z '' ']' + command -v aws + echo 'AWS CLI is installed' AWS CLI is installed + command -v gsutil + echo 'gutil is installed' gutil is installed + command -v az + echo 'Azure CLI is installed' Azure CLI is installed ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ grep aws ++ head -n1 ++ cut -d: -f1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.g5CoPTfC1E +++ mktemp ++ local LAST_ERR=/tmp/tmp.2W7zM9qXMO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.g5CoPTfC1E ++ cat /tmp/tmp.2W7zM9qXMO ++ rm /tmp/tmp.g5CoPTfC1E /tmp/tmp.2W7zM9qXMO ++ return 0 + FIRST_AWS_BACKUP=cron-scheduled-backup-aws-s3-20264945757-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-aws-s3-20264945757-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.47WCBjIuN7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Zv2dQYgN0r ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.47WCBjIuN7 ++ cat /tmp/tmp.Zv2dQYgN0r ++ rm /tmp/tmp.47WCBjIuN7 /tmp/tmp.Zv2dQYgN0r ++ return 0 + JOB_AWS_BACKUP=xb-cron-scheduled-backup-aws-s3-20264945757-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-aws-s3-20264945757 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FCqYIOw5Lz +++ mktemp ++ local LAST_ERR=/tmp/tmp.CmGtrSt0uc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FCqYIOw5Lz ++ cat /tmp/tmp.CmGtrSt0uc ++ rm /tmp/tmp.FCqYIOw5Lz /tmp/tmp.CmGtrSt0uc ++ return 0 + POD_AWS_BACKUP=xb-cron-scheduled-backup-aws-s3-20264945757-q6fav-8s6nr ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep gcp +++ mktemp ++ cut -d: -f1 ++ grep Succeeded ++ head -n1 ++ local LAST_OUT=/tmp/tmp.A6LgoZDEkG +++ mktemp ++ local LAST_ERR=/tmp/tmp.fANCFNoPvQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A6LgoZDEkG ++ cat /tmp/tmp.fANCFNoPvQ ++ rm /tmp/tmp.A6LgoZDEkG /tmp/tmp.fANCFNoPvQ ++ return 0 + FIRST_GCP_BACKUP=cron-scheduled-backup-gcp-cs-2026495157-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-gcp-cs-2026495157-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jfhzCPtKqJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.HC2DEmkrCD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jfhzCPtKqJ ++ cat /tmp/tmp.HC2DEmkrCD ++ rm /tmp/tmp.jfhzCPtKqJ /tmp/tmp.HC2DEmkrCD ++ return 0 + JOB_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-2026495157-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-gcp-cs-2026495157 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oF8KaakPPW +++ mktemp ++ local LAST_ERR=/tmp/tmp.BzVENJxZT4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oF8KaakPPW ++ cat /tmp/tmp.BzVENJxZT4 ++ rm /tmp/tmp.oF8KaakPPW /tmp/tmp.BzVENJxZT4 ++ return 0 + POD_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-2026495157-q6fav-mkmf6 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ grep azure ++ head -n1 +++ mktemp ++ cut -d: -f1 ++ local LAST_OUT=/tmp/tmp.9LHIxGq7eh +++ mktemp ++ local LAST_ERR=/tmp/tmp.gEnYKcbG27 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9LHIxGq7eh ++ cat /tmp/tmp.gEnYKcbG27 ++ rm /tmp/tmp.9LHIxGq7eh /tmp/tmp.gEnYKcbG27 ++ return 0 + FIRST_AZURE_BACKUP=cron-scheduled-backup-azure-blob-2026495348-q6fav ++ kubectl_bin get jobs ++ awk '{print $1}' +++ mktemp ++ grep cron-scheduled-backup-azure-blob-2026495348-q6fav ++ local LAST_OUT=/tmp/tmp.WXkj8Io1CR +++ mktemp ++ local LAST_ERR=/tmp/tmp.F0hwpEt8Ak ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WXkj8Io1CR ++ cat /tmp/tmp.F0hwpEt8Ak ++ rm /tmp/tmp.WXkj8Io1CR /tmp/tmp.F0hwpEt8Ak ++ return 0 + JOB_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-2026495348-q6fav ++ kubectl_bin get pods ++ awk '{print $1}' +++ mktemp ++ grep xb-cron-scheduled-backup-azure-blob-2026495348 ++ local LAST_OUT=/tmp/tmp.Kqzfwl1Uqy +++ mktemp ++ local LAST_ERR=/tmp/tmp.SAuxzSCQPX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Kqzfwl1Uqy ++ cat /tmp/tmp.SAuxzSCQPX ++ rm /tmp/tmp.Kqzfwl1Uqy /tmp/tmp.SAuxzSCQPX ++ return 0 + POD_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-2026495348-q6fav-jhgg4 ++ kubectl_bin get pxc-backup cron-scheduled-backup-aws-s3-20264945757-q6fav -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ cut -c 6- +++ mktemp ++ local LAST_OUT=/tmp/tmp.TcehUD8jKF +++ mktemp ++ local LAST_ERR=/tmp/tmp.zsMhivf73r ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-aws-s3-20264945757-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TcehUD8jKF ++ cat /tmp/tmp.zsMhivf73r ++ rm /tmp/tmp.TcehUD8jKF /tmp/tmp.zsMhivf73r ++ return 0 + BACKUP_DEST_AWS=operator-testing/scheduled-backup-2026-04-09-04:57:57-full ++ kubectl_bin get pxc-backup cron-scheduled-backup-gcp-cs-2026495157-q6fav -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' +++ mktemp ++ cut -c 6- ++ local LAST_OUT=/tmp/tmp.VgJGoTN38C +++ mktemp ++ local LAST_ERR=/tmp/tmp.FHCuiYCAdt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-gcp-cs-2026495157-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VgJGoTN38C ++ cat /tmp/tmp.FHCuiYCAdt ++ rm /tmp/tmp.VgJGoTN38C /tmp/tmp.FHCuiYCAdt ++ return 0 + BACKUP_DEST_GCP=operator-testing/scheduled-backup-2026-04-09-05:01:57-full ++ kubectl_bin get pxc-backup cron-scheduled-backup-azure-blob-2026495348-q6fav -o 'jsonpath={.status.destination}' ++ cut -c 9- ++ sed -e 's/.json$//' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Y5OE4WpISe +++ mktemp ++ local LAST_ERR=/tmp/tmp.CGujhGyqtZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-azure-blob-2026495348-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Y5OE4WpISe ++ cat /tmp/tmp.CGujhGyqtZ ++ rm /tmp/tmp.Y5OE4WpISe /tmp/tmp.CGujhGyqtZ ++ return 0 + BACKUP_DEST_AZURE=operator-testing/scheduled-backup-2026-04-09-05:03:48-full + desc 'Check backup existence' + set +o xtrace ----------------------------------------------------------------------------------- Check backup existence ----------------------------------------------------------------------------------- + setup_aws_credentials + local secret_name=aws-s3-secret + [[ -n AKIARXP3OARBMEQ6FSEW ]] + [[ -n gljUukL3/TuQnzrtJUgiURH6vqjt+SEo89jGS2Z/ ]] + echo 'AWS credentials already set in environment' AWS credentials already set in environment + return 0 + check_backup_existence_aws operator-testing/scheduled-backup-2026-04-09-04:57:57-full .sst_info/sst_info.00000000000000000000 ++ echo operator-testing/scheduled-backup-2026-04-09-04:57:57-full ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/scheduled-backup-2026-04-09-04:57:57-full ++ cut -d/ -f2- + key_prefix=scheduled-backup-2026-04-09-04:57:57-full + key=.sst_info/sst_info.00000000000000000000 + storage_name=aws-s3 + retry=0 + aws s3api head-object --bucket operator-testing --key scheduled-backup-2026-04-09-04:57:57-full.sst_info/sst_info.00000000000000000000 + echo 'Backup scheduled-backup-2026-04-09-04:57:57-full.sst_info/sst_info.00000000000000000000 found in bucket operator-testing in aws-s3' Backup scheduled-backup-2026-04-09-04:57:57-full.sst_info/sst_info.00000000000000000000 found in bucket operator-testing in aws-s3 + setup_gcs_credentials + local secret_name=gcp-cs-secret + gsutil ls + echo 'Setting up GCS credentials from K8s secret: gcp-cs-secret' Setting up GCS credentials from K8s secret: gcp-cs-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'GCS credentials configured successfully' GCS credentials configured successfully + check_backup_existence_gcs operator-testing/scheduled-backup-2026-04-09-05:01:57-full + backup_dest_gcp=operator-testing/scheduled-backup-2026-04-09-05:01:57-full + storage_name=gcp-cs + retry=0 + key=.sst_info/sst_info.00000000000000000000 + gcs_path=gs://operator-testing/scheduled-backup-2026-04-09-05:01:57-full.sst_info/sst_info.00000000000000000000 + gsutil ls gs://operator-testing/scheduled-backup-2026-04-09-05:01:57-full.sst_info/sst_info.00000000000000000000 + echo 'Backup found in gcp-cs: gs://operator-testing/scheduled-backup-2026-04-09-05:01:57-full.sst_info/sst_info.00000000000000000000' Backup found in gcp-cs: gs://operator-testing/scheduled-backup-2026-04-09-05:01:57-full.sst_info/sst_info.00000000000000000000 + setup_azure_credentials + local secret_name=azure-secret + echo 'Setting up Azure credentials from K8s secret: azure-secret' Setting up Azure credentials from K8s secret: azure-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'Azure credentials configured successfully' Azure credentials configured successfully + check_backup_existence_azure operator-testing/scheduled-backup-2026-04-09-05:03:48-full .sst_info/sst_info.00000000000000000000 ++ echo operator-testing/scheduled-backup-2026-04-09-05:03:48-full ++ cut -d/ -f1 + container=operator-testing ++ cut -d/ -f2- ++ echo operator-testing/scheduled-backup-2026-04-09-05:03:48-full + blob_prefix=scheduled-backup-2026-04-09-05:03:48-full + blob=.sst_info/sst_info.00000000000000000000 + storage_name=azure-blob + retry=0 + blob_path=scheduled-backup-2026-04-09-05:03:48-full.sst_info/sst_info.00000000000000000000 + az storage blob show --container-name operator-testing --name scheduled-backup-2026-04-09-05:03:48-full.sst_info/sst_info.00000000000000000000 + echo 'Backup scheduled-backup-2026-04-09-05:03:48-full.sst_info/sst_info.00000000000000000000 found in container operator-testing in azure-blob' Backup scheduled-backup-2026-04-09-05:03:48-full.sst_info/sst_info.00000000000000000000 found in container operator-testing in azure-blob + desc 'Check that KEEP option saves correct backup'\''s amount (1 for our settings)' + set +o xtrace ----------------------------------------------------------------------------------- Check that KEEP option saves correct backup's amount (1 for our settings) ----------------------------------------------------------------------------------- + desc 'add backups schedule for gcs storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for gcs storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.5NhlKhXy6V + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.V6zD3eQasV + local exit_status=0 + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ seq 0 2 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5NhlKhXy6V perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.V6zD3eQasV + rm /tmp/tmp.5NhlKhXy6V /tmp/tmp.V6zD3eQasV + return 0 + sleep 50 + wait_num_backups gcp-cs 2 + local storage=gcp-cs + local num=2 + local timer=0 + echo 'Starting to check number 2 of backups for gcp-cs' Starting to check number 2 of backups for gcp-cs ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.YxxRxI4ULb +++ mktemp ++ local LAST_ERR=/tmp/tmp.m2Fln2qzkI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YxxRxI4ULb ++ cat /tmp/tmp.m2Fln2qzkI ++ rm /tmp/tmp.YxxRxI4ULb /tmp/tmp.m2Fln2qzkI ++ return 0 + [[ 1 -ne 2 ]] + [[ 0 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.kwp33GvxYo +++ mktemp ++ local LAST_ERR=/tmp/tmp.azlRro69cH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kwp33GvxYo ++ cat /tmp/tmp.azlRro69cH ++ rm /tmp/tmp.kwp33GvxYo /tmp/tmp.azlRro69cH ++ return 0 + [[ 1 -ne 2 ]] + [[ 1 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.ddIHS2csTJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.LjTtgFZGgR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ddIHS2csTJ ++ cat /tmp/tmp.LjTtgFZGgR ++ rm /tmp/tmp.ddIHS2csTJ /tmp/tmp.LjTtgFZGgR ++ return 0 + [[ 1 -ne 2 ]] + [[ 2 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.yCCKZnof02 +++ mktemp ++ local LAST_ERR=/tmp/tmp.fpey5wRpHS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yCCKZnof02 ++ cat /tmp/tmp.fpey5wRpHS ++ rm /tmp/tmp.yCCKZnof02 /tmp/tmp.fpey5wRpHS ++ return 0 + [[ 1 -ne 2 ]] + [[ 3 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.nVHzkIJ3JB +++ mktemp ++ local LAST_ERR=/tmp/tmp.T3U1hMon3X ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nVHzkIJ3JB ++ cat /tmp/tmp.T3U1hMon3X ++ rm /tmp/tmp.nVHzkIJ3JB /tmp/tmp.T3U1hMon3X ++ return 0 + [[ 1 -ne 2 ]] + [[ 4 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs +++ mktemp ++ wc -l ++ grep -E :Succeeded ++ local LAST_OUT=/tmp/tmp.zhkdmSTW7M +++ mktemp ++ local LAST_ERR=/tmp/tmp.ECECAW90GS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zhkdmSTW7M ++ cat /tmp/tmp.ECECAW90GS ++ rm /tmp/tmp.zhkdmSTW7M /tmp/tmp.ECECAW90GS ++ return 0 + [[ 1 -ne 2 ]] + [[ 5 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.1ZGo2oXMVm +++ mktemp ++ local LAST_ERR=/tmp/tmp.PzhrfXzRKA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1ZGo2oXMVm ++ cat /tmp/tmp.PzhrfXzRKA ++ rm /tmp/tmp.1ZGo2oXMVm /tmp/tmp.PzhrfXzRKA ++ return 0 + [[ 1 -ne 2 ]] + [[ 6 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs +++ mktemp ++ wc -l ++ grep -E :Succeeded ++ local LAST_OUT=/tmp/tmp.BcqrskxqLT +++ mktemp ++ local LAST_ERR=/tmp/tmp.2PpQRpHdr6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BcqrskxqLT ++ cat /tmp/tmp.2PpQRpHdr6 ++ rm /tmp/tmp.BcqrskxqLT /tmp/tmp.2PpQRpHdr6 ++ return 0 + [[ 1 -ne 2 ]] + [[ 7 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs +++ mktemp ++ wc -l ++ grep -E :Succeeded ++ local LAST_OUT=/tmp/tmp.Tu75AxQ8Wg +++ mktemp ++ local LAST_ERR=/tmp/tmp.xRktYMAzq1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Tu75AxQ8Wg ++ cat /tmp/tmp.xRktYMAzq1 ++ rm /tmp/tmp.Tu75AxQ8Wg /tmp/tmp.xRktYMAzq1 ++ return 0 + [[ 1 -ne 2 ]] + [[ 8 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.lqmhVn80ME +++ mktemp ++ local LAST_ERR=/tmp/tmp.fzgbrFYKMk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lqmhVn80ME ++ cat /tmp/tmp.fzgbrFYKMk ++ rm /tmp/tmp.lqmhVn80ME /tmp/tmp.fzgbrFYKMk ++ return 0 + [[ 1 -ne 2 ]] + [[ 9 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.bdMHiGF37I +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZwBKrZfuWV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bdMHiGF37I ++ cat /tmp/tmp.ZwBKrZfuWV ++ rm /tmp/tmp.bdMHiGF37I /tmp/tmp.ZwBKrZfuWV ++ return 0 + [[ 1 -ne 2 ]] + [[ 10 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.qgDcfwFr9w +++ mktemp ++ local LAST_ERR=/tmp/tmp.k7debrJDum ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qgDcfwFr9w ++ cat /tmp/tmp.k7debrJDum ++ rm /tmp/tmp.qgDcfwFr9w /tmp/tmp.k7debrJDum ++ return 0 + [[ 1 -ne 2 ]] + [[ 11 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.9BwKjeLSy6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.qbmns8W7n9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9BwKjeLSy6 ++ cat /tmp/tmp.qbmns8W7n9 ++ rm /tmp/tmp.9BwKjeLSy6 /tmp/tmp.qbmns8W7n9 ++ return 0 + [[ 1 -ne 2 ]] + [[ 12 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ grep gcp-cs ++ local LAST_OUT=/tmp/tmp.hAChtD2Skt +++ mktemp ++ local LAST_ERR=/tmp/tmp.OQ6cjAH6GY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hAChtD2Skt ++ cat /tmp/tmp.OQ6cjAH6GY ++ rm /tmp/tmp.hAChtD2Skt /tmp/tmp.OQ6cjAH6GY ++ return 0 + [[ 1 -ne 2 ]] + [[ 13 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.x8y9afn29c +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZOSrZwsWj9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.x8y9afn29c ++ cat /tmp/tmp.ZOSrZwsWj9 ++ rm /tmp/tmp.x8y9afn29c /tmp/tmp.ZOSrZwsWj9 ++ return 0 + [[ 1 -ne 2 ]] + [[ 14 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.2xpxpYooNK +++ mktemp ++ local LAST_ERR=/tmp/tmp.Qm14UGr8bo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2xpxpYooNK ++ cat /tmp/tmp.Qm14UGr8bo ++ rm /tmp/tmp.2xpxpYooNK /tmp/tmp.Qm14UGr8bo ++ return 0 + [[ 1 -ne 2 ]] + [[ 15 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.AcNummJqaJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.virvJKvbDR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AcNummJqaJ ++ cat /tmp/tmp.virvJKvbDR ++ rm /tmp/tmp.AcNummJqaJ /tmp/tmp.virvJKvbDR ++ return 0 + [[ 1 -ne 2 ]] + [[ 16 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.2UOLPsRNn5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.0EVZKbSOrF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2UOLPsRNn5 ++ cat /tmp/tmp.0EVZKbSOrF ++ rm /tmp/tmp.2UOLPsRNn5 /tmp/tmp.0EVZKbSOrF ++ return 0 + [[ 2 -ne 2 ]] + echo 'Finished to check number 2 of backups for gcp-cs. Timer: 17' Finished to check number 2 of backups for gcp-cs. Timer: 17 + echo 'Patch backup for yearly backup for gcs storage' Patch backup for yearly backup for gcs storage + kubectl patch pxc scheduled-backup --type=merge -p '{"spec":{"backup":{"schedule":[{"name":"each-min-gcp-cs","storageName":"gcp-cs","keep":1,"schedule":"0 0 1 * *"}]}}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched + wait_num_backups gcp-cs 1 + local storage=gcp-cs + local num=1 + local timer=0 + echo 'Starting to check number 1 of backups for gcp-cs' Starting to check number 1 of backups for gcp-cs ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.qlNnfoFE9a +++ mktemp ++ local LAST_ERR=/tmp/tmp.soWJ9l7b2v ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qlNnfoFE9a ++ cat /tmp/tmp.soWJ9l7b2v ++ rm /tmp/tmp.qlNnfoFE9a /tmp/tmp.soWJ9l7b2v ++ return 0 + [[ 2 -ne 1 ]] + [[ 0 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.JZZ9yZsyFV +++ mktemp ++ local LAST_ERR=/tmp/tmp.eD9yoj0k3W ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JZZ9yZsyFV ++ cat /tmp/tmp.eD9yoj0k3W ++ rm /tmp/tmp.JZZ9yZsyFV /tmp/tmp.eD9yoj0k3W ++ return 0 + [[ 2 -ne 1 ]] + [[ 1 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.dWRrQnyVhH +++ mktemp ++ local LAST_ERR=/tmp/tmp.n3d8ywWk51 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dWRrQnyVhH ++ cat /tmp/tmp.n3d8ywWk51 ++ rm /tmp/tmp.dWRrQnyVhH /tmp/tmp.n3d8ywWk51 ++ return 0 + [[ 2 -ne 1 ]] + [[ 2 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.RDrsh0swc8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.2kS8a7kq5p ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RDrsh0swc8 ++ cat /tmp/tmp.2kS8a7kq5p ++ rm /tmp/tmp.RDrsh0swc8 /tmp/tmp.2kS8a7kq5p ++ return 0 + [[ 2 -ne 1 ]] + [[ 3 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.IpU1jC6djh +++ mktemp ++ local LAST_ERR=/tmp/tmp.7oZObQASeA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IpU1jC6djh ++ cat /tmp/tmp.7oZObQASeA ++ rm /tmp/tmp.IpU1jC6djh /tmp/tmp.7oZObQASeA ++ return 0 + [[ 2 -ne 1 ]] + [[ 4 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.X12zUauV8L +++ mktemp ++ local LAST_ERR=/tmp/tmp.LyrHi09Y0n ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.X12zUauV8L ++ cat /tmp/tmp.LyrHi09Y0n ++ rm /tmp/tmp.X12zUauV8L /tmp/tmp.LyrHi09Y0n ++ return 0 + [[ 2 -ne 1 ]] + [[ 5 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs +++ mktemp ++ grep -E :Succeeded ++ wc -l ++ local LAST_OUT=/tmp/tmp.gszc6ALJ0k +++ mktemp ++ local LAST_ERR=/tmp/tmp.4QVTUvTq7a ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gszc6ALJ0k ++ cat /tmp/tmp.4QVTUvTq7a ++ rm /tmp/tmp.gszc6ALJ0k /tmp/tmp.4QVTUvTq7a ++ return 0 + [[ 2 -ne 1 ]] + [[ 6 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.k8UeGQ7Veb +++ mktemp ++ local LAST_ERR=/tmp/tmp.jtczWvRKZy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.k8UeGQ7Veb ++ cat /tmp/tmp.jtczWvRKZy ++ rm /tmp/tmp.k8UeGQ7Veb /tmp/tmp.jtczWvRKZy ++ return 0 + [[ 1 -ne 1 ]] + echo 'Finished to check number 1 of backups for gcp-cs. Timer: 7' Finished to check number 1 of backups for gcp-cs. Timer: 7 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + kubectl_bin apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_OUT=/tmp/tmp.961VNSgCmw + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local LAST_ERR=/tmp/tmp.139EbiZF5R + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.961VNSgCmw perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.139EbiZF5R + rm /tmp/tmp.961VNSgCmw /tmp/tmp.139EbiZF5R + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AQgvgGeoVp +++ mktemp ++ local LAST_ERR=/tmp/tmp.zIC2i3KSol ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AQgvgGeoVp ++ cat /tmp/tmp.zIC2i3KSol ++ rm /tmp/tmp.AQgvgGeoVp /tmp/tmp.zIC2i3KSol ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.YLzWzClm7T +++ mktemp ++ local LAST_ERR=/tmp/tmp.5s6ilW2X2o ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YLzWzClm7T ++ cat /tmp/tmp.5s6ilW2X2o ++ rm /tmp/tmp.YLzWzClm7T /tmp/tmp.5s6ilW2X2o ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for azure storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for azure storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_OUT=/tmp/tmp.oCS4SjUhg6 ++ mktemp + local LAST_ERR=/tmp/tmp.PquemIBME1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oCS4SjUhg6 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.PquemIBME1 + rm /tmp/tmp.oCS4SjUhg6 /tmp/tmp.PquemIBME1 + return 0 + sleep 50 + wait_num_backups azure-blob 2 + local storage=azure-blob + local num=2 + local timer=0 + echo 'Starting to check number 2 of backups for azure-blob' Starting to check number 2 of backups for azure-blob ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.OxcAxFvml5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.s8G00lNkS5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OxcAxFvml5 ++ cat /tmp/tmp.s8G00lNkS5 ++ rm /tmp/tmp.OxcAxFvml5 /tmp/tmp.s8G00lNkS5 ++ return 0 + [[ 1 -ne 2 ]] + [[ 0 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.x5JQlsKPR4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.g7Gpg7yHPC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.x5JQlsKPR4 ++ cat /tmp/tmp.g7Gpg7yHPC ++ rm /tmp/tmp.x5JQlsKPR4 /tmp/tmp.g7Gpg7yHPC ++ return 0 + [[ 1 -ne 2 ]] + [[ 1 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.fxGf90Y01j +++ mktemp ++ local LAST_ERR=/tmp/tmp.CaGD534o5l ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fxGf90Y01j ++ cat /tmp/tmp.CaGD534o5l ++ rm /tmp/tmp.fxGf90Y01j /tmp/tmp.CaGD534o5l ++ return 0 + [[ 1 -ne 2 ]] + [[ 2 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.5OE04GniZ7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.dTj7owvogv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5OE04GniZ7 ++ cat /tmp/tmp.dTj7owvogv ++ rm /tmp/tmp.5OE04GniZ7 /tmp/tmp.dTj7owvogv ++ return 0 + [[ 1 -ne 2 ]] + [[ 3 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.E7zSILxXgp +++ mktemp ++ local LAST_ERR=/tmp/tmp.conscYC7fs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.E7zSILxXgp ++ cat /tmp/tmp.conscYC7fs ++ rm /tmp/tmp.E7zSILxXgp /tmp/tmp.conscYC7fs ++ return 0 + [[ 1 -ne 2 ]] + [[ 4 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -E :Succeeded ++ grep azure-blob ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Itukh6lyku +++ mktemp ++ local LAST_ERR=/tmp/tmp.QRhaBv562x ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Itukh6lyku ++ cat /tmp/tmp.QRhaBv562x ++ rm /tmp/tmp.Itukh6lyku /tmp/tmp.QRhaBv562x ++ return 0 + [[ 1 -ne 2 ]] + [[ 5 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ grep azure-blob ++ local LAST_OUT=/tmp/tmp.mS3MWmZO1T +++ mktemp ++ local LAST_ERR=/tmp/tmp.PRX334AvVp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mS3MWmZO1T ++ cat /tmp/tmp.PRX334AvVp ++ rm /tmp/tmp.mS3MWmZO1T /tmp/tmp.PRX334AvVp ++ return 0 + [[ 1 -ne 2 ]] + [[ 6 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ wc -l ++ grep -E :Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.8Qnqnhmo7E +++ mktemp ++ local LAST_ERR=/tmp/tmp.COj7XZyzE6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8Qnqnhmo7E ++ cat /tmp/tmp.COj7XZyzE6 ++ rm /tmp/tmp.8Qnqnhmo7E /tmp/tmp.COj7XZyzE6 ++ return 0 + [[ 1 -ne 2 ]] + [[ 7 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.52yKJcqrmd +++ mktemp ++ local LAST_ERR=/tmp/tmp.v0P20A2FQQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.52yKJcqrmd ++ cat /tmp/tmp.v0P20A2FQQ ++ rm /tmp/tmp.52yKJcqrmd /tmp/tmp.v0P20A2FQQ ++ return 0 + [[ 1 -ne 2 ]] + [[ 8 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.g2qMgMHA1c +++ mktemp ++ local LAST_ERR=/tmp/tmp.jh6X7Sq2FJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.g2qMgMHA1c ++ cat /tmp/tmp.jh6X7Sq2FJ ++ rm /tmp/tmp.g2qMgMHA1c /tmp/tmp.jh6X7Sq2FJ ++ return 0 + [[ 1 -ne 2 ]] + [[ 9 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob +++ mktemp ++ wc -l ++ grep -E :Succeeded ++ local LAST_OUT=/tmp/tmp.cHv5beao6R +++ mktemp ++ local LAST_ERR=/tmp/tmp.BjaqUCVlFR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cHv5beao6R ++ cat /tmp/tmp.BjaqUCVlFR ++ rm /tmp/tmp.cHv5beao6R /tmp/tmp.BjaqUCVlFR ++ return 0 + [[ 1 -ne 2 ]] + [[ 10 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.OI8Cm1Bojo +++ mktemp ++ local LAST_ERR=/tmp/tmp.nMR7uF7niv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OI8Cm1Bojo ++ cat /tmp/tmp.nMR7uF7niv ++ rm /tmp/tmp.OI8Cm1Bojo /tmp/tmp.nMR7uF7niv ++ return 0 + [[ 1 -ne 2 ]] + [[ 11 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.aQspeuor1z +++ mktemp ++ local LAST_ERR=/tmp/tmp.8jztnRISLZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aQspeuor1z ++ cat /tmp/tmp.8jztnRISLZ ++ rm /tmp/tmp.aQspeuor1z /tmp/tmp.8jztnRISLZ ++ return 0 + [[ 1 -ne 2 ]] + [[ 12 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.w7ZmLZfRy8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.BxcMhCFv8M ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.w7ZmLZfRy8 ++ cat /tmp/tmp.BxcMhCFv8M ++ rm /tmp/tmp.w7ZmLZfRy8 /tmp/tmp.BxcMhCFv8M ++ return 0 + [[ 1 -ne 2 ]] + [[ 13 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.gikLKZ9uEI +++ mktemp ++ local LAST_ERR=/tmp/tmp.aUQVNNdcVr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gikLKZ9uEI ++ cat /tmp/tmp.aUQVNNdcVr ++ rm /tmp/tmp.gikLKZ9uEI /tmp/tmp.aUQVNNdcVr ++ return 0 + [[ 1 -ne 2 ]] + [[ 14 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.7jbHmcyVjS +++ mktemp ++ local LAST_ERR=/tmp/tmp.U7SP2krW5d ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7jbHmcyVjS ++ cat /tmp/tmp.U7SP2krW5d ++ rm /tmp/tmp.7jbHmcyVjS /tmp/tmp.U7SP2krW5d ++ return 0 + [[ 1 -ne 2 ]] + [[ 15 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.OHxCFpFFjn +++ mktemp ++ local LAST_ERR=/tmp/tmp.fbIUHe9NTf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OHxCFpFFjn ++ cat /tmp/tmp.fbIUHe9NTf ++ rm /tmp/tmp.OHxCFpFFjn /tmp/tmp.fbIUHe9NTf ++ return 0 + [[ 1 -ne 2 ]] + [[ 16 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.PN0rEesN4B +++ mktemp ++ local LAST_ERR=/tmp/tmp.b1ZIY8kMpT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PN0rEesN4B ++ cat /tmp/tmp.b1ZIY8kMpT ++ rm /tmp/tmp.PN0rEesN4B /tmp/tmp.b1ZIY8kMpT ++ return 0 + [[ 2 -ne 2 ]] + echo 'Finished to check number 2 of backups for azure-blob. Timer: 17' Finished to check number 2 of backups for azure-blob. Timer: 17 + echo 'Patch backup for yearly backup for azure storag' Patch backup for yearly backup for azure storag + kubectl patch pxc scheduled-backup --type=merge -p '{"spec":{"backup":{"schedule":[{"name":"each-min-azure-every","storageName":"azure-blob","keep":1,"schedule":"0 0 1 * *"}]}}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched + wait_num_backups azure-blob 1 + local storage=azure-blob + local num=1 + local timer=0 + echo 'Starting to check number 1 of backups for azure-blob' Starting to check number 1 of backups for azure-blob ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ wc -l +++ mktemp ++ grep -E :Succeeded ++ local LAST_OUT=/tmp/tmp.qkAELl2HDR +++ mktemp ++ local LAST_ERR=/tmp/tmp.o3uT8RFDI4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qkAELl2HDR ++ cat /tmp/tmp.o3uT8RFDI4 ++ rm /tmp/tmp.qkAELl2HDR /tmp/tmp.o3uT8RFDI4 ++ return 0 + [[ 2 -ne 1 ]] + [[ 0 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.bsGlm98nYd +++ mktemp ++ local LAST_ERR=/tmp/tmp.oAx5nhLZfR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bsGlm98nYd ++ cat /tmp/tmp.oAx5nhLZfR ++ rm /tmp/tmp.bsGlm98nYd /tmp/tmp.oAx5nhLZfR ++ return 0 + [[ 2 -ne 1 ]] + [[ 1 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.QyGPTtUhfJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZZuTh8Tqby ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QyGPTtUhfJ ++ cat /tmp/tmp.ZZuTh8Tqby ++ rm /tmp/tmp.QyGPTtUhfJ /tmp/tmp.ZZuTh8Tqby ++ return 0 + [[ 2 -ne 1 ]] + [[ 2 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.iRrTzlPO2I +++ mktemp ++ local LAST_ERR=/tmp/tmp.cL9ur2t8PI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iRrTzlPO2I ++ cat /tmp/tmp.cL9ur2t8PI ++ rm /tmp/tmp.iRrTzlPO2I /tmp/tmp.cL9ur2t8PI ++ return 0 + [[ 2 -ne 1 ]] + [[ 3 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob +++ mktemp ++ grep -E :Succeeded ++ wc -l ++ local LAST_OUT=/tmp/tmp.xpoTpdeXI5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4foRnb2f8D ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xpoTpdeXI5 ++ cat /tmp/tmp.4foRnb2f8D ++ rm /tmp/tmp.xpoTpdeXI5 /tmp/tmp.4foRnb2f8D ++ return 0 + [[ 2 -ne 1 ]] + [[ 4 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.O7gC7JAShg +++ mktemp ++ local LAST_ERR=/tmp/tmp.WnDHopcGM7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.O7gC7JAShg ++ cat /tmp/tmp.WnDHopcGM7 ++ rm /tmp/tmp.O7gC7JAShg /tmp/tmp.WnDHopcGM7 ++ return 0 + [[ 2 -ne 1 ]] + [[ 5 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.OpgFFZlORP +++ mktemp ++ local LAST_ERR=/tmp/tmp.90fv3Pnatx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OpgFFZlORP ++ cat /tmp/tmp.90fv3Pnatx ++ rm /tmp/tmp.OpgFFZlORP /tmp/tmp.90fv3Pnatx ++ return 0 + [[ 2 -ne 1 ]] + [[ 6 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.fNS7yQuCQQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.pWDylNEhPX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fNS7yQuCQQ ++ cat /tmp/tmp.pWDylNEhPX ++ rm /tmp/tmp.fNS7yQuCQQ /tmp/tmp.pWDylNEhPX ++ return 0 + [[ 1 -ne 1 ]] + echo 'Finished to check number 1 of backups for azure-blob. Timer: 7' Finished to check number 1 of backups for azure-blob. Timer: 7 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.gq374I0xF5 + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + local LAST_ERR=/tmp/tmp.5osSPolwNr + local exit_status=0 + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gq374I0xF5 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.5osSPolwNr + rm /tmp/tmp.gq374I0xF5 /tmp/tmp.5osSPolwNr + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.yyONQz1y6C +++ mktemp ++ local LAST_ERR=/tmp/tmp.RcICqDK2xY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yyONQz1y6C ++ cat /tmp/tmp.RcICqDK2xY ++ rm /tmp/tmp.yyONQz1y6C /tmp/tmp.RcICqDK2xY ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.X2hCcvJTRL +++ mktemp ++ local LAST_ERR=/tmp/tmp.eVbtp96gyC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.X2hCcvJTRL ++ cat /tmp/tmp.eVbtp96gyC ++ rm /tmp/tmp.X2hCcvJTRL /tmp/tmp.eVbtp96gyC ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + local LAST_OUT=/tmp/tmp.MrFXQof3cQ + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + local LAST_ERR=/tmp/tmp.9nyX4t0CHo + local exit_status=0 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MrFXQof3cQ perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.9nyX4t0CHo + rm /tmp/tmp.MrFXQof3cQ /tmp/tmp.9nyX4t0CHo + return 0 + sleep 50 + wait_num_backups aws-s3 2 + local storage=aws-s3 + local num=2 + local timer=0 + echo 'Starting to check number 2 of backups for aws-s3' Starting to check number 2 of backups for aws-s3 ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.OCTyMiAvOG +++ mktemp ++ local LAST_ERR=/tmp/tmp.pEl4OIBW8A ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OCTyMiAvOG ++ cat /tmp/tmp.pEl4OIBW8A ++ rm /tmp/tmp.OCTyMiAvOG /tmp/tmp.pEl4OIBW8A ++ return 0 + [[ 1 -ne 2 ]] + [[ 0 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 +++ mktemp ++ grep -E :Succeeded ++ wc -l ++ local LAST_OUT=/tmp/tmp.fcHlR8fHVT +++ mktemp ++ local LAST_ERR=/tmp/tmp.hsT4LKE7Qg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fcHlR8fHVT ++ cat /tmp/tmp.hsT4LKE7Qg ++ rm /tmp/tmp.fcHlR8fHVT /tmp/tmp.hsT4LKE7Qg ++ return 0 + [[ 1 -ne 2 ]] + [[ 1 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ grep aws-s3 ++ grep -E :Succeeded ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.U6pIPRsWPZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.bARIJB8UzF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.U6pIPRsWPZ ++ cat /tmp/tmp.bARIJB8UzF ++ rm /tmp/tmp.U6pIPRsWPZ /tmp/tmp.bARIJB8UzF ++ return 0 + [[ 1 -ne 2 ]] + [[ 2 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.VVJT0fH2gA +++ mktemp ++ local LAST_ERR=/tmp/tmp.MW6l8JUbhN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VVJT0fH2gA ++ cat /tmp/tmp.MW6l8JUbhN ++ rm /tmp/tmp.VVJT0fH2gA /tmp/tmp.MW6l8JUbhN ++ return 0 + [[ 1 -ne 2 ]] + [[ 3 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.oWap9dBl6e +++ mktemp ++ local LAST_ERR=/tmp/tmp.rZ687n1b0k ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oWap9dBl6e ++ cat /tmp/tmp.rZ687n1b0k ++ rm /tmp/tmp.oWap9dBl6e /tmp/tmp.rZ687n1b0k ++ return 0 + [[ 1 -ne 2 ]] + [[ 4 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.6sBUTvLLcO +++ mktemp ++ local LAST_ERR=/tmp/tmp.FjDPk6u0xn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6sBUTvLLcO ++ cat /tmp/tmp.FjDPk6u0xn ++ rm /tmp/tmp.6sBUTvLLcO /tmp/tmp.FjDPk6u0xn ++ return 0 + [[ 1 -ne 2 ]] + [[ 5 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.iYUdHyHlSV +++ mktemp ++ local LAST_ERR=/tmp/tmp.nrSc5A6Sv6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iYUdHyHlSV ++ cat /tmp/tmp.nrSc5A6Sv6 ++ rm /tmp/tmp.iYUdHyHlSV /tmp/tmp.nrSc5A6Sv6 ++ return 0 + [[ 1 -ne 2 ]] + [[ 6 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.E5rIyn0ngx +++ mktemp ++ local LAST_ERR=/tmp/tmp.H3evC6PsGx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.E5rIyn0ngx ++ cat /tmp/tmp.H3evC6PsGx ++ rm /tmp/tmp.E5rIyn0ngx /tmp/tmp.H3evC6PsGx ++ return 0 + [[ 1 -ne 2 ]] + [[ 7 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.5VIswufznq +++ mktemp ++ local LAST_ERR=/tmp/tmp.Y1iRy0nKUQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5VIswufznq ++ cat /tmp/tmp.Y1iRy0nKUQ ++ rm /tmp/tmp.5VIswufznq /tmp/tmp.Y1iRy0nKUQ ++ return 0 + [[ 1 -ne 2 ]] + [[ 8 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ wc -l ++ grep -E :Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.2rSxuyONSj +++ mktemp ++ local LAST_ERR=/tmp/tmp.JWqDmSkTZ2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2rSxuyONSj ++ cat /tmp/tmp.JWqDmSkTZ2 ++ rm /tmp/tmp.2rSxuyONSj /tmp/tmp.JWqDmSkTZ2 ++ return 0 + [[ 1 -ne 2 ]] + [[ 9 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.nSdSQ0ebaG +++ mktemp ++ local LAST_ERR=/tmp/tmp.NMa7rTgnNx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nSdSQ0ebaG ++ cat /tmp/tmp.NMa7rTgnNx ++ rm /tmp/tmp.nSdSQ0ebaG /tmp/tmp.NMa7rTgnNx ++ return 0 + [[ 1 -ne 2 ]] + [[ 10 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 +++ mktemp ++ grep -E :Succeeded ++ wc -l ++ local LAST_OUT=/tmp/tmp.n3Xc1LBhwe +++ mktemp ++ local LAST_ERR=/tmp/tmp.prfBnOJyBb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.n3Xc1LBhwe ++ cat /tmp/tmp.prfBnOJyBb ++ rm /tmp/tmp.n3Xc1LBhwe /tmp/tmp.prfBnOJyBb ++ return 0 + [[ 1 -ne 2 ]] + [[ 11 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.R6WW6KfUEh +++ mktemp ++ local LAST_ERR=/tmp/tmp.PovQsQTC70 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.R6WW6KfUEh ++ cat /tmp/tmp.PovQsQTC70 ++ rm /tmp/tmp.R6WW6KfUEh /tmp/tmp.PovQsQTC70 ++ return 0 + [[ 1 -ne 2 ]] + [[ 12 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.XWZgNHnMHq +++ mktemp ++ local LAST_ERR=/tmp/tmp.pMQ3yPZr7Y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XWZgNHnMHq ++ cat /tmp/tmp.pMQ3yPZr7Y ++ rm /tmp/tmp.XWZgNHnMHq /tmp/tmp.pMQ3yPZr7Y ++ return 0 + [[ 1 -ne 2 ]] + [[ 13 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.pvYNG7Z2Pd +++ mktemp ++ local LAST_ERR=/tmp/tmp.REO092ch0c ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pvYNG7Z2Pd ++ cat /tmp/tmp.REO092ch0c ++ rm /tmp/tmp.pvYNG7Z2Pd /tmp/tmp.REO092ch0c ++ return 0 + [[ 1 -ne 2 ]] + [[ 14 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.uWoIdyK1Go +++ mktemp ++ local LAST_ERR=/tmp/tmp.GIchDvmJf8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uWoIdyK1Go ++ cat /tmp/tmp.GIchDvmJf8 ++ rm /tmp/tmp.uWoIdyK1Go /tmp/tmp.GIchDvmJf8 ++ return 0 + [[ 1 -ne 2 ]] + [[ 15 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.iZt4GijX9D +++ mktemp ++ local LAST_ERR=/tmp/tmp.QWVfIBgKEc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iZt4GijX9D ++ cat /tmp/tmp.QWVfIBgKEc ++ rm /tmp/tmp.iZt4GijX9D /tmp/tmp.QWVfIBgKEc ++ return 0 + [[ 1 -ne 2 ]] + [[ 16 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 +++ mktemp ++ grep -E :Succeeded ++ wc -l ++ local LAST_OUT=/tmp/tmp.GpxqUrnwmN +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZbaojjBLMF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GpxqUrnwmN ++ cat /tmp/tmp.ZbaojjBLMF ++ rm /tmp/tmp.GpxqUrnwmN /tmp/tmp.ZbaojjBLMF ++ return 0 + [[ 2 -ne 2 ]] + echo 'Finished to check number 2 of backups for aws-s3. Timer: 17' Finished to check number 2 of backups for aws-s3. Timer: 17 + echo 'Patch backup for yearly backup for aws s3 storage' Patch backup for yearly backup for aws s3 storage + kubectl patch pxc scheduled-backup --type=merge -p '{"spec":{"backup":{"schedule":[{"name":"each-min-aws-s3","storageName":"aws-s3","keep":1,"schedule":"0 0 1 * *"}]}}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched + wait_num_backups aws-s3 1 + local storage=aws-s3 + local num=1 + local timer=0 + echo 'Starting to check number 1 of backups for aws-s3' Starting to check number 1 of backups for aws-s3 ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.gZvatNlysT +++ mktemp ++ local LAST_ERR=/tmp/tmp.EeNe8aBjIk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gZvatNlysT ++ cat /tmp/tmp.EeNe8aBjIk ++ rm /tmp/tmp.gZvatNlysT /tmp/tmp.EeNe8aBjIk ++ return 0 + [[ 2 -ne 1 ]] + [[ 0 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Rkohdlkhzm +++ mktemp ++ local LAST_ERR=/tmp/tmp.GJJE0Un3uq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Rkohdlkhzm ++ cat /tmp/tmp.GJJE0Un3uq ++ rm /tmp/tmp.Rkohdlkhzm /tmp/tmp.GJJE0Un3uq ++ return 0 + [[ 2 -ne 1 ]] + [[ 1 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.4VKQFpiBxs +++ mktemp ++ local LAST_ERR=/tmp/tmp.PcMBo7KIb6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4VKQFpiBxs ++ cat /tmp/tmp.PcMBo7KIb6 ++ rm /tmp/tmp.4VKQFpiBxs /tmp/tmp.PcMBo7KIb6 ++ return 0 + [[ 2 -ne 1 ]] + [[ 2 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.PBs7pQpgbQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.ugwvwgWlql ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PBs7pQpgbQ ++ cat /tmp/tmp.ugwvwgWlql ++ rm /tmp/tmp.PBs7pQpgbQ /tmp/tmp.ugwvwgWlql ++ return 0 + [[ 2 -ne 1 ]] + [[ 3 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.8yKnyYZ1C2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.hYEXcOamAu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8yKnyYZ1C2 ++ cat /tmp/tmp.hYEXcOamAu ++ rm /tmp/tmp.8yKnyYZ1C2 /tmp/tmp.hYEXcOamAu ++ return 0 + [[ 2 -ne 1 ]] + [[ 4 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ wc -l ++ grep -E :Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.LPUEtERGsd +++ mktemp ++ local LAST_ERR=/tmp/tmp.kgBLPfsgft ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LPUEtERGsd ++ cat /tmp/tmp.kgBLPfsgft ++ rm /tmp/tmp.LPUEtERGsd /tmp/tmp.kgBLPfsgft ++ return 0 + [[ 2 -ne 1 ]] + [[ 5 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.VifskBRjiS +++ mktemp ++ local LAST_ERR=/tmp/tmp.B8iaiIbLGj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VifskBRjiS ++ cat /tmp/tmp.B8iaiIbLGj ++ rm /tmp/tmp.VifskBRjiS /tmp/tmp.B8iaiIbLGj ++ return 0 + [[ 2 -ne 1 ]] + [[ 6 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.m5yHxNNxqb +++ mktemp ++ local LAST_ERR=/tmp/tmp.xGrf6Zwj7R ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.m5yHxNNxqb ++ cat /tmp/tmp.xGrf6Zwj7R ++ rm /tmp/tmp.m5yHxNNxqb /tmp/tmp.xGrf6Zwj7R ++ return 0 + [[ 2 -ne 1 ]] + [[ 7 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.L53ReQcJ11 +++ mktemp ++ local LAST_ERR=/tmp/tmp.yD6BuVWxdc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.L53ReQcJ11 ++ cat /tmp/tmp.yD6BuVWxdc ++ rm /tmp/tmp.L53ReQcJ11 /tmp/tmp.yD6BuVWxdc ++ return 0 + [[ 2 -ne 1 ]] + [[ 8 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.IUmhmroeiA +++ mktemp ++ local LAST_ERR=/tmp/tmp.q5yYCMTRE2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IUmhmroeiA ++ cat /tmp/tmp.q5yYCMTRE2 ++ rm /tmp/tmp.IUmhmroeiA /tmp/tmp.q5yYCMTRE2 ++ return 0 + [[ 1 -ne 1 ]] + echo 'Finished to check number 1 of backups for aws-s3. Timer: 9' Finished to check number 1 of backups for aws-s3. Timer: 9 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.inpYy4reef + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + local LAST_ERR=/tmp/tmp.Rtzb9DTssf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.inpYy4reef perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.Rtzb9DTssf + rm /tmp/tmp.inpYy4reef /tmp/tmp.Rtzb9DTssf + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.8b5S9Pkm6S +++ mktemp ++ local LAST_ERR=/tmp/tmp.jAk8YzBHp7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8b5S9Pkm6S ++ cat /tmp/tmp.jAk8YzBHp7 ++ rm /tmp/tmp.8b5S9Pkm6S /tmp/tmp.jAk8YzBHp7 ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.9IGC3Gt6gX +++ mktemp ++ local LAST_ERR=/tmp/tmp.2RDm2mHjq5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9IGC3Gt6gX ++ cat /tmp/tmp.2RDm2mHjq5 ++ rm /tmp/tmp.9IGC3Gt6gX /tmp/tmp.2RDm2mHjq5 ++ return 0 + [[ 0 -gt 0 ]] + sleep 30 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ grep aws ++ head -n1 +++ mktemp ++ cut -d: -f1 ++ local LAST_OUT=/tmp/tmp.BAPKm9vreg +++ mktemp ++ local LAST_ERR=/tmp/tmp.l1rhgNIY98 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BAPKm9vreg ++ cat /tmp/tmp.l1rhgNIY98 ++ rm /tmp/tmp.BAPKm9vreg /tmp/tmp.l1rhgNIY98 ++ return 0 + SECOND_AWS_BACKUP=cron-scheduled-backup-aws-s3-20264951022-q6fav ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ grep gcp ++ cut -d: -f1 ++ head -n1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.N8ErNH2S0q +++ mktemp ++ local LAST_ERR=/tmp/tmp.087Bwp4fHe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.N8ErNH2S0q ++ cat /tmp/tmp.087Bwp4fHe ++ rm /tmp/tmp.N8ErNH2S0q /tmp/tmp.087Bwp4fHe ++ return 0 + SECOND_GCP_BACKUP=cron-scheduled-backup-gcp-cs-202649560-q6fav ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ head -n1 ++ grep azure +++ mktemp ++ cut -d: -f1 ++ local LAST_OUT=/tmp/tmp.aETq8AkCuR +++ mktemp ++ local LAST_ERR=/tmp/tmp.LcNr9ynxuD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aETq8AkCuR ++ cat /tmp/tmp.LcNr9ynxuD ++ rm /tmp/tmp.aETq8AkCuR /tmp/tmp.LcNr9ynxuD ++ return 0 + SECOND_AZURE_BACKUP=cron-scheduled-backup-azure-blob-2026495812-q6fav + [[ cron-scheduled-backup-gcp-cs-2026495157-q6fav == \c\r\o\n\-\s\c\h\e\d\u\l\e\d\-\b\a\c\k\u\p\-\g\c\p\-\c\s\-\2\0\2\6\4\9\5\6\0\-\q\6\f\a\v ]] + [[ cron-scheduled-backup-azure-blob-2026495348-q6fav == \c\r\o\n\-\s\c\h\e\d\u\l\e\d\-\b\a\c\k\u\p\-\a\z\u\r\e\-\b\l\o\b\-\2\0\2\6\4\9\5\8\1\2\-\q\6\f\a\v ]] + [[ cron-scheduled-backup-aws-s3-20264945757-q6fav == \c\r\o\n\-\s\c\h\e\d\u\l\e\d\-\b\a\c\k\u\p\-\a\w\s\-\s\3\-\2\0\2\6\4\9\5\1\0\2\2\-\q\6\f\a\v ]] + desc 'Check backup deletion' + set +o xtrace ----------------------------------------------------------------------------------- Check backup deletion ----------------------------------------------------------------------------------- + check_backup_deletion_aws operator-testing/scheduled-backup-2026-04-09-04:57:57-full .sst_info/sst_info.00000000000000000000 ++ echo operator-testing/scheduled-backup-2026-04-09-04:57:57-full ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/scheduled-backup-2026-04-09-04:57:57-full ++ cut -d/ -f2- + key_prefix=scheduled-backup-2026-04-09-04:57:57-full + key=.sst_info/sst_info.00000000000000000000 + storage_name=aws-s3 + retry=0 + aws s3api head-object --bucket operator-testing --key scheduled-backup-2026-04-09-04:57:57-full.sst_info/sst_info.00000000000000000000 + echo 'Backup scheduled-backup-2026-04-09-04:57:57-full.sst_info/sst_info.00000000000000000000 in bucket operator-testing not found in aws-s3' Backup scheduled-backup-2026-04-09-04:57:57-full.sst_info/sst_info.00000000000000000000 in bucket operator-testing not found in aws-s3 + check_backup_deletion_gcs operator-testing/scheduled-backup-2026-04-09-05:01:57-full + backup_dest_gcp=operator-testing/scheduled-backup-2026-04-09-05:01:57-full + storage_name=gcp-cs + retry=0 + key=.sst_info/sst_info.00000000000000000000 + gcs_path=gs://operator-testing/scheduled-backup-2026-04-09-05:01:57-full.sst_info/sst_info.00000000000000000000 + gsutil ls gs://operator-testing/scheduled-backup-2026-04-09-05:01:57-full.sst_info/sst_info.00000000000000000000 + echo 'Backup gs://operator-testing/scheduled-backup-2026-04-09-05:01:57-full.sst_info/sst_info.00000000000000000000 not found in gcp-cs' Backup gs://operator-testing/scheduled-backup-2026-04-09-05:01:57-full.sst_info/sst_info.00000000000000000000 not found in gcp-cs + check_backup_deletion_azure operator-testing/scheduled-backup-2026-04-09-05:03:48-full .sst_info/sst_info.00000000000000000000 ++ echo operator-testing/scheduled-backup-2026-04-09-05:03:48-full ++ cut -d/ -f1 + container=operator-testing ++ echo operator-testing/scheduled-backup-2026-04-09-05:03:48-full ++ cut -d/ -f2- + blob_prefix=scheduled-backup-2026-04-09-05:03:48-full + blob=.sst_info/sst_info.00000000000000000000 + storage_name=azure-blob + retry=0 + blob_path=scheduled-backup-2026-04-09-05:03:48-full.sst_info/sst_info.00000000000000000000 + az storage blob show --container-name operator-testing --name scheduled-backup-2026-04-09-05:03:48-full.sst_info/sst_info.00000000000000000000 + echo 'Backup scheduled-backup-2026-04-09-05:03:48-full.sst_info/sst_info.00000000000000000000 in container operator-testing not found in azure-blob' Backup scheduled-backup-2026-04-09-05:03:48-full.sst_info/sst_info.00000000000000000000 in container operator-testing not found in azure-blob ++ get_backup_name pvc ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :pvc:Succeeded ++ tail -1 ++ cut -d : -f 1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.dDX6O8RgT1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.PlQcdUB8RY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dDX6O8RgT1 ++ cat /tmp/tmp.PlQcdUB8RY ++ rm /tmp/tmp.dDX6O8RgT1 /tmp/tmp.PlQcdUB8RY ++ return 0 + backup_name_pvc=cron-scheduled-backup-pvc-2026494562-q6fav ++ get_backup_name minio ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :minio:Succeeded +++ mktemp ++ cut -d : -f 1 ++ tail -1 ++ local LAST_OUT=/tmp/tmp.UeLTLPdrTZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.XYoubKX3yu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UeLTLPdrTZ ++ cat /tmp/tmp.XYoubKX3yu ++ rm /tmp/tmp.UeLTLPdrTZ /tmp/tmp.XYoubKX3yu ++ return 0 + backup_name_minio=cron-scheduled-backup-minio-20264945948-q6fav + '[' -z '' ']' ++ get_backup_name aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :aws-s3:Succeeded ++ tail -1 ++ cut -d : -f 1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.3w0qpOdgMt +++ mktemp ++ local LAST_ERR=/tmp/tmp.q8A6mFoe6J ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3w0qpOdgMt ++ cat /tmp/tmp.q8A6mFoe6J ++ rm /tmp/tmp.3w0qpOdgMt /tmp/tmp.q8A6mFoe6J ++ return 0 + backup_name_aws=cron-scheduled-backup-aws-s3-20264951022-q6fav ++ get_backup_name gcp-cs ++ grep :gcp-cs:Succeeded ++ cut -d : -f 1 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ tail -1 ++ local LAST_OUT=/tmp/tmp.zhcQsy6PO8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.WKghpBChIE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zhcQsy6PO8 ++ cat /tmp/tmp.WKghpBChIE ++ rm /tmp/tmp.zhcQsy6PO8 /tmp/tmp.WKghpBChIE ++ return 0 + backup_name_gcp=cron-scheduled-backup-gcp-cs-202649560-q6fav ++ get_backup_name azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :azure-blob:Succeeded +++ mktemp ++ tail -1 ++ cut -d : -f 1 ++ local LAST_OUT=/tmp/tmp.Vc3WtJAbn1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.N39DaaDP3E ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Vc3WtJAbn1 ++ cat /tmp/tmp.N39DaaDP3E ++ rm /tmp/tmp.Vc3WtJAbn1 /tmp/tmp.N39DaaDP3E ++ return 0 + backup_name_azure=cron-scheduled-backup-azure-blob-2026495812-q6fav + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + kubectl_bin apply -f - + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.ob8OoRpnYp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-22500~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_ERR=/tmp/tmp.faFzYhqGik + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ob8OoRpnYp perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.faFzYhqGik + rm /tmp/tmp.ob8OoRpnYp /tmp/tmp.faFzYhqGik + return 0 + run_recovery_check scheduled-backup cron-scheduled-backup-pvc-2026494562-q6fav + local cluster=scheduled-backup + local backup1=cron-scheduled-backup-pvc-2026494562-q6fav + restore_name=pvc-2026494562-q6fav + desc 'write data after backup' + set +o xtrace ----------------------------------------------------------------------------------- write data after backup ----------------------------------------------------------------------------------- + run_mysql 'INSERT myApp.myApp (id) VALUES (100501)' '-h scheduled-backup-proxysql -uroot -proot_password' + local 'command=INSERT myApp.myApp (id) VALUES (100501)' + local 'uri=-h scheduled-backup-proxysql -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ISvnPa8L0p +++ mktemp ++ local LAST_ERR=/tmp/tmp.XOhLw0zpt0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ISvnPa8L0p ++ cat /tmp/tmp.XOhLw0zpt0 ++ rm /tmp/tmp.ISvnPa8L0p /tmp/tmp.XOhLw0zpt0 ++ return 0 + client_pod=pxc-client-97cb9c68b-k2wtj + wait_pod pxc-client-97cb9c68b-k2wtj + local pod=pxc-client-97cb9c68b-k2wtj + local max_retry=480 + local ns= ++ echo pxc-client-97cb9c68b-k2wtj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-97cb9c68b-k2wtj condition met waiting for pod/pxc-client-97cb9c68b-k2wtj to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 20 + compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-2-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zlF7xwRX99 +++ mktemp ++ local LAST_ERR=/tmp/tmp.9G5E87TOTM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zlF7xwRX99 ++ cat /tmp/tmp.9G5E87TOTM ++ rm /tmp/tmp.zlF7xwRX99 /tmp/tmp.9G5E87TOTM ++ return 0 + client_pod=pxc-client-97cb9c68b-k2wtj + wait_pod pxc-client-97cb9c68b-k2wtj + local pod=pxc-client-97cb9c68b-k2wtj + local max_retry=480 + local ns= ++ echo pxc-client-97cb9c68b-k2wtj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-97cb9c68b-k2wtj condition met waiting for pod/pxc-client-97cb9c68b-k2wtj to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.ipBQjx3O3C/select-2.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.ipBQjx3O3C/select-2.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-2.sql /tmp/tmp.ipBQjx3O3C/select-2.sql + compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-2-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.J29yQS7Up7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Tdfx91EpNf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.J29yQS7Up7 ++ cat /tmp/tmp.Tdfx91EpNf ++ rm /tmp/tmp.J29yQS7Up7 /tmp/tmp.Tdfx91EpNf ++ return 0 + client_pod=pxc-client-97cb9c68b-k2wtj + wait_pod pxc-client-97cb9c68b-k2wtj + local pod=pxc-client-97cb9c68b-k2wtj + local max_retry=480 + local ns= ++ echo pxc-client-97cb9c68b-k2wtj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-97cb9c68b-k2wtj condition met waiting for pod/pxc-client-97cb9c68b-k2wtj to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.ipBQjx3O3C/select-2.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.ipBQjx3O3C/select-2.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-2.sql /tmp/tmp.ipBQjx3O3C/select-2.sql + compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-2-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UJPzJRvJ1X +++ mktemp ++ local LAST_ERR=/tmp/tmp.4wf6Prsufg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UJPzJRvJ1X ++ cat /tmp/tmp.4wf6Prsufg ++ rm /tmp/tmp.UJPzJRvJ1X /tmp/tmp.4wf6Prsufg ++ return 0 + client_pod=pxc-client-97cb9c68b-k2wtj + wait_pod pxc-client-97cb9c68b-k2wtj + local pod=pxc-client-97cb9c68b-k2wtj + local max_retry=480 + local ns= ++ echo pxc-client-97cb9c68b-k2wtj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-97cb9c68b-k2wtj condition met waiting for pod/pxc-client-97cb9c68b-k2wtj to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.ipBQjx3O3C/select-2.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.ipBQjx3O3C/select-2.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/scheduled-backup/compare/select-2.sql /tmp/tmp.ipBQjx3O3C/select-2.sql + desc 'recover backup' + set +o xtrace ----------------------------------------------------------------------------------- recover backup ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/deploy/backup/restore.yaml + /usr/bin/sed 's/backupName: .*/backupName: cron-scheduled-backup-pvc-2026494562-q6fav/' + /usr/bin/sed 's/name: .*/name: pvc-2026494562-q6fav/' + /usr/bin/sed 's/pxcCluster: .*/pxcCluster: scheduled-backup/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.r9HGdhNP2G ++ mktemp + local LAST_ERR=/tmp/tmp.A0gLrNqYEj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.r9HGdhNP2G perconaxtradbclusterrestore.pxc.percona.com/pvc-2026494562-q6fav created + cat /tmp/tmp.A0gLrNqYEj + rm /tmp/tmp.r9HGdhNP2G /tmp/tmp.A0gLrNqYEj + return 0 + wait_backup_restore pvc-2026494562-q6fav + local backup_name=pvc-2026494562-q6fav + local target_state=Succeeded + local wait_time=720 + set +o xtrace waiting for pxc-restore/pvc-2026494562-q6fav to reach Succeeded state 2026-04-09T05:13:03 pxc-restore/pvc-2026494562-q6fav state: Starting 2026-04-09T05:13:05 pxc-restore/pvc-2026494562-q6fav state: Starting 2026-04-09T05:13:07 pxc-restore/pvc-2026494562-q6fav state: Starting 2026-04-09T05:13:09 pxc-restore/pvc-2026494562-q6fav state: Starting 2026-04-09T05:13:11 pxc-restore/pvc-2026494562-q6fav state: Starting 2026-04-09T05:13:12 pxc-restore/pvc-2026494562-q6fav state: Starting 2026-04-09T05:13:14 pxc-restore/pvc-2026494562-q6fav state: Starting 2026-04-09T05:13:16 pxc-restore/pvc-2026494562-q6fav state: Starting 2026-04-09T05:13:18 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:20 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:22 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:23 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:25 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:27 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:29 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:30 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:32 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:34 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:36 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:37 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:39 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:40 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:42 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:44 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:46 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:48 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:50 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:51 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:52 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:54 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:56 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:57 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:13:59 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:14:01 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:14:03 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:14:05 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:14:07 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:14:08 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:14:09 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:14:11 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:14:13 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:14:15 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:14:17 pxc-restore/pvc-2026494562-q6fav state: Stopping Cluster 2026-04-09T05:14:18 pxc-restore/pvc-2026494562-q6fav state: Restoring 2026-04-09T05:14:20 pxc-restore/pvc-2026494562-q6fav state: Restoring 2026-04-09T05:14:22 pxc-restore/pvc-2026494562-q6fav state: Restoring 2026-04-09T05:14:24 pxc-restore/pvc-2026494562-q6fav state: Restoring 2026-04-09T05:14:26 pxc-restore/pvc-2026494562-q6fav state: Restoring 2026-04-09T05:14:28 pxc-restore/pvc-2026494562-q6fav state: Restoring 2026-04-09T05:14:30 pxc-restore/pvc-2026494562-q6fav state: Restoring 2026-04-09T05:14:32 pxc-restore/pvc-2026494562-q6fav state: Restoring 2026-04-09T05:14:34 pxc-restore/pvc-2026494562-q6fav state: Restoring 2026-04-09T05:14:36 pxc-restore/pvc-2026494562-q6fav state: Restoring 2026-04-09T05:14:38 pxc-restore/pvc-2026494562-q6fav state: Restoring 2026-04-09T05:14:39 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:14:41 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:14:42 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:14:44 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:14:46 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:14:48 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:14:49 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:14:50 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:14:52 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:14:54 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:14:56 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:14:57 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:14:59 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:00 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:02 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:04 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:06 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:07 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:09 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:10 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:12 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:14 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:16 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:18 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:20 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:22 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:23 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:24 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:26 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:27 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:29 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:31 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:32 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:33 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:35 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:37 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:39 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:41 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:43 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:45 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:47 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:49 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:51 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:53 pxc-restore/pvc-2026494562-q6fav state: Preparing Cluster 2026-04-09T05:15:55 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:15:57 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:15:59 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:01 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:03 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:05 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:07 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:09 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:11 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:12 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:14 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:15 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:17 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:19 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:21 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:23 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:25 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:27 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:29 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:31 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:33 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:35 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:37 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:39 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:41 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:43 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster 2026-04-09T05:16:45 pxc-restore/pvc-2026494562-q6fav state: Starting Cluster