Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/logs/scheduled-backup-8-0.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + main + create_infra scheduled-backup-28307 + local ns=scheduled-backup-28307 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n scheduled-backup-8515 scheduled-backup --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.nX1reoO1Wl ++ mktemp + local LAST_ERR=/tmp/tmp.wypLD4lOmD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nX1reoO1Wl perconaxtradbcluster.pxc.percona.com "scheduled-backup" deleted from scheduled-backup-8515 namespace + cat /tmp/tmp.wypLD4lOmD + rm /tmp/tmp.nX1reoO1Wl /tmp/tmp.wypLD4lOmD + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.AngetNc5Cl ++ mktemp + local LAST_ERR=/tmp/tmp.xpqUHv1y3X + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AngetNc5Cl perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-aws-s3-202651621911-q6fav" deleted from scheduled-backup-8515 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-azure-blob-202651621714-q6fav" deleted from scheduled-backup-8515 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-gcp-cs-20265162159-q6fav" deleted from scheduled-backup-8515 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-minio-2026516205848-q6fav" deleted from scheduled-backup-8515 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-pvc-2026516205456-q6fav" deleted from scheduled-backup-8515 namespace + cat /tmp/tmp.xpqUHv1y3X + rm /tmp/tmp.AngetNc5Cl /tmp/tmp.xpqUHv1y3X + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.AlqQsnQhNd ++ mktemp + local LAST_ERR=/tmp/tmp.tw6GuDqZs1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AlqQsnQhNd perconaxtradbclusterrestore.pxc.percona.com "aws-s3-202651621911-q6fav" deleted from scheduled-backup-8515 namespace perconaxtradbclusterrestore.pxc.percona.com "minio-2026516205848-q6fav" deleted from scheduled-backup-8515 namespace perconaxtradbclusterrestore.pxc.percona.com "pvc-2026516205456-q6fav" deleted from scheduled-backup-8515 namespace + cat /tmp/tmp.tw6GuDqZs1 + rm /tmp/tmp.AlqQsnQhNd /tmp/tmp.tw6GuDqZs1 + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.6Id1TU1uw6 ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.CLp4P59qCx + local LAST_ERR=/tmp/tmp.40svtl9tbn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_ERR=/tmp/tmp.AKsDjz9H8I + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6Id1TU1uw6 + cat /tmp/tmp.40svtl9tbn + rm /tmp/tmp.6Id1TU1uw6 /tmp/tmp.40svtl9tbn + return 0 namespace "scheduled-backup-8515" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CLp4P59qCx namespace "pxc-operator" deleted + cat /tmp/tmp.AKsDjz9H8I + rm /tmp/tmp.CLp4P59qCx /tmp/tmp.AKsDjz9H8I + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.NdotQNXcHq ++ mktemp + local LAST_ERR=/tmp/tmp.iE7vA5cTf2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NdotQNXcHq namespace/pxc-operator created + cat /tmp/tmp.iE7vA5cTf2 + rm /tmp/tmp.NdotQNXcHq /tmp/tmp.iE7vA5cTf2 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.o2eBc7G0jQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.P60jbtv9aC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.o2eBc7G0jQ ++ cat /tmp/tmp.P60jbtv9aC ++ rm /tmp/tmp.o2eBc7G0jQ /tmp/tmp.P60jbtv9aC ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2467-3dc7f023-16-cluster9 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.GymRExWAjv ++ mktemp + local LAST_ERR=/tmp/tmp.f4yCNuu0nD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2467-3dc7f023-16-cluster9 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GymRExWAjv Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2467-3dc7f023-16-cluster9" modified. + cat /tmp/tmp.f4yCNuu0nD + rm /tmp/tmp.GymRExWAjv /tmp/tmp.f4yCNuu0nD + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.ox0nQ1zJ1Z ++ mktemp + local LAST_ERR=/tmp/tmp.xepZtwRNby + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ox0nQ1zJ1Z customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.xepZtwRNby + rm /tmp/tmp.ox0nQ1zJ1Z /tmp/tmp.xepZtwRNby + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.4eofCIPDHy ++ mktemp + local LAST_ERR=/tmp/tmp.r5vsAcx5cs + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4eofCIPDHy clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.r5vsAcx5cs + rm /tmp/tmp.4eofCIPDHy /tmp/tmp.r5vsAcx5cs + return 0 + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/deploy/cw-operator.yaml + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.c9YMd0uWfl ++ mktemp + local LAST_ERR=/tmp/tmp.ryQCjWyWVv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.c9YMd0uWfl deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.ryQCjWyWVv + rm /tmp/tmp.c9YMd0uWfl /tmp/tmp.ryQCjWyWVv + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.O4xFQf25A9 ++ mktemp + local LAST_ERR=/tmp/tmp.SMgoauBDpG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.O4xFQf25A9 pod/percona-xtradb-cluster-operator-9d9fbdb5-d8f4r condition met + cat /tmp/tmp.SMgoauBDpG E0516 21:40:56.723075 29645 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-9d9fbdb5-d8f4r&resourceVersion=1778967656279642000&timeoutSeconds=427&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.O4xFQf25A9 /tmp/tmp.SMgoauBDpG + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.GPaSsN8E2u +++ mktemp ++ local LAST_ERR=/tmp/tmp.7Ciyn9aLkL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GPaSsN8E2u ++ cat /tmp/tmp.7Ciyn9aLkL ++ rm /tmp/tmp.GPaSsN8E2u /tmp/tmp.7Ciyn9aLkL ++ return 0 + wait_pod percona-xtradb-cluster-operator-9d9fbdb5-d8f4r 480 pxc-operator + local pod=percona-xtradb-cluster-operator-9d9fbdb5-d8f4r + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-9d9fbdb5-d8f4r ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-9d9fbdb5-d8f4r condition met E0516 21:41:04.127614 30524 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-9d9fbdb5-d8f4r&resourceVersion=1778967660316046000&timeoutSeconds=573&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-9d9fbdb5-d8f4r to become Ready.Ok + sleep 3 + create_namespace scheduled-backup-28307 + local namespace=scheduled-backup-28307 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ tail -n1 ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces scheduled-backup-28307' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces scheduled-backup-28307 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace scheduled-backup-28307 + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.oEmuRmBdWD ++ mktemp + local LAST_ERR=/tmp/tmp.kSrYEmOGoN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-28307 ++ mktemp + local LAST_OUT=/tmp/tmp.ley2j7ryhO ++ mktemp + local LAST_ERR=/tmp/tmp.4gVvFT2gEs + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-28307 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ley2j7ryhO + cat /tmp/tmp.4gVvFT2gEs + rm /tmp/tmp.ley2j7ryhO /tmp/tmp.4gVvFT2gEs + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-28307 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.oEmuRmBdWD + cat /tmp/tmp.kSrYEmOGoN Error from server (NotFound): namespaces "scheduled-backup-28307" not found + rm /tmp/tmp.oEmuRmBdWD /tmp/tmp.kSrYEmOGoN + return 1 + : + wait_for_delete namespace/scheduled-backup-28307 + local res=namespace/scheduled-backup-28307 + echo -n 'waiting for namespace/scheduled-backup-28307 to be deleted' waiting for namespace/scheduled-backup-28307 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "scheduled-backup-28307" not found + desc 'create namespace scheduled-backup-28307' + set +o xtrace ----------------------------------------------------------------------------------- create namespace scheduled-backup-28307 ----------------------------------------------------------------------------------- + kubectl_bin create namespace scheduled-backup-28307 ++ mktemp + local LAST_OUT=/tmp/tmp.F3m8C6RZH5 ++ mktemp + local LAST_ERR=/tmp/tmp.aiqhFkDz7m + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace scheduled-backup-28307 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.F3m8C6RZH5 namespace/scheduled-backup-28307 created + cat /tmp/tmp.aiqhFkDz7m + rm /tmp/tmp.F3m8C6RZH5 /tmp/tmp.aiqhFkDz7m + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.U4jCSr41Yf +++ mktemp ++ local LAST_ERR=/tmp/tmp.8EORtlxXTv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.U4jCSr41Yf ++ cat /tmp/tmp.8EORtlxXTv ++ rm /tmp/tmp.U4jCSr41Yf /tmp/tmp.8EORtlxXTv ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2467-3dc7f023-16-cluster9 --namespace=scheduled-backup-28307 ++ mktemp + local LAST_OUT=/tmp/tmp.IaMBGoLazg ++ mktemp + local LAST_ERR=/tmp/tmp.ScaSVf4HQA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2467-3dc7f023-16-cluster9 --namespace=scheduled-backup-28307 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IaMBGoLazg Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2467-3dc7f023-16-cluster9" modified. + cat /tmp/tmp.ScaSVf4HQA + rm /tmp/tmp.IaMBGoLazg /tmp/tmp.ScaSVf4HQA + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.vrbxNTeso3 ++ mktemp + local LAST_ERR=/tmp/tmp.wZTn195feZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vrbxNTeso3 secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.wZTn195feZ + rm /tmp/tmp.vrbxNTeso3 /tmp/tmp.wZTn195feZ + return 0 + start_minio + deploy_helm scheduled-backup-28307 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Sat May 16 21:41:45 2026 NAMESPACE: scheduled-backup-28307 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.scheduled-backup-28307.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-28307 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-28307 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-28307 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-28307 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GrrsqkfePa +++ mktemp ++ local LAST_ERR=/tmp/tmp.tZ2qNsGZlQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GrrsqkfePa ++ cat /tmp/tmp.tZ2qNsGZlQ ++ rm /tmp/tmp.GrrsqkfePa /tmp/tmp.tZ2qNsGZlQ ++ return 0 + MINIO_POD=minio-service-5fd5489bdc-c6zbn + wait_pod minio-service-5fd5489bdc-c6zbn + local pod=minio-service-5fd5489bdc-c6zbn + local max_retry=480 + local ns= ++ echo minio-service-5fd5489bdc-c6zbn ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minio-service-5fd5489bdc-c6zbn condition met E0516 21:42:47.434250 11357 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dminio-service-5fd5489bdc-c6zbn&resourceVersion=1778967765731156000&timeoutSeconds=314&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/minio-service-5fd5489bdc-c6zbn to become Ready.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.zXHQ5ETYUM ++ mktemp + local LAST_ERR=/tmp/tmp.Fjr9todcV3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zXHQ5ETYUM make_bucket: operator-testing pod "aws-cli" deleted from scheduled-backup-28307 namespace + cat /tmp/tmp.Fjr9todcV3 All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.zXHQ5ETYUM /tmp/tmp.Fjr9todcV3 + return 0 + cluster=scheduled-backup + cat - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.HSYg8HgAca ++ mktemp + local LAST_ERR=/tmp/tmp.cguQBSDPt6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HSYg8HgAca priorityclass.scheduling.k8s.io/high-priority configured + cat /tmp/tmp.cguQBSDPt6 + rm /tmp/tmp.HSYg8HgAca /tmp/tmp.cguQBSDPt6 + return 0 + spinup_pxc scheduled-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local cluster=scheduled-backup + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.49im9N1izg ++ mktemp + local LAST_ERR=/tmp/tmp.CZYXsj8Qb3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.49im9N1izg secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.CZYXsj8Qb3 + rm /tmp/tmp.49im9N1izg /tmp/tmp.CZYXsj8Qb3 + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/conf/client.yml + local pvc_name= ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.6Fo7LJeyoH ++ mktemp + local LAST_ERR=/tmp/tmp.QHGGKZu7S5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6Fo7LJeyoH deployment.apps/pxc-client created + cat /tmp/tmp.QHGGKZu7S5 + rm /tmp/tmp.6Fo7LJeyoH /tmp/tmp.QHGGKZu7S5 + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + local LAST_OUT=/tmp/tmp.di1WOfb4qY ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_ERR=/tmp/tmp.ZYHg9gCB2y + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.di1WOfb4qY perconaxtradbcluster.pxc.percona.com/scheduled-backup created + cat /tmp/tmp.ZYHg9gCB2y + rm /tmp/tmp.di1WOfb4qY /tmp/tmp.ZYHg9gCB2y + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy scheduled-backup ++ local target_cluster=scheduled-backup +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GXeRQKCyDI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cwcl6znTeu +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.GXeRQKCyDI +++ cat /tmp/tmp.cwcl6znTeu +++ rm /tmp/tmp.GXeRQKCyDI /tmp/tmp.cwcl6znTeu +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Cj78BJZtfo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FLWmJ3fL23 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Cj78BJZtfo +++ cat /tmp/tmp.FLWmJ3fL23 +++ rm /tmp/tmp.Cj78BJZtfo /tmp/tmp.FLWmJ3fL23 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scheduled-backup-proxysql ++ return + local proxy=scheduled-backup-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-28307 ++ mktemp + local LAST_OUT=/tmp/tmp.iHUe4Jk8Sc ++ mktemp + local LAST_ERR=/tmp/tmp.irNRRjkTno + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-28307 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-28307 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-28307 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.iHUe4Jk8Sc + cat /tmp/tmp.irNRRjkTno error: no matching resources found + rm /tmp/tmp.iHUe4Jk8Sc /tmp/tmp.irNRRjkTno + return 1 + true + wait_for_running scheduled-backup-proxysql 1 + local name=scheduled-backup-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-proxysql-0 480 + local pod=scheduled-backup-proxysql-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/scheduled-backup-proxysql-0 condition met waiting for pod/scheduled-backup-proxysql-0 to become Ready.Ok + wait_for_running scheduled-backup-pxc 3 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 480 + local pod=scheduled-backup-pxc-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 480 + local pod=scheduled-backup-pxc-1 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 480 + local pod=scheduled-backup-pxc-2 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yPLrApuMNX +++ mktemp ++ local LAST_ERR=/tmp/tmp.r7o0G3Aks9 ++ local exit_status=0 +++ seq 0 2 ++ base64 --decode ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yPLrApuMNX ++ cat /tmp/tmp.r7o0G3Aks9 ++ rm /tmp/tmp.yPLrApuMNX /tmp/tmp.r7o0G3Aks9 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zxi8xBzAEg +++ mktemp ++ local LAST_ERR=/tmp/tmp.usKsfeG0ZD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zxi8xBzAEg ++ cat /tmp/tmp.usKsfeG0ZD ++ rm /tmp/tmp.zxi8xBzAEg /tmp/tmp.usKsfeG0ZD ++ return 0 + client_pod=pxc-client-67fc4995bb-qtkdl + wait_pod pxc-client-67fc4995bb-qtkdl + local pod=pxc-client-67fc4995bb-qtkdl + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-qtkdl ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-qtkdl condition met E0516 21:47:29.337242 10905 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-qtkdl&resourceVersion=1778968047471824000&timeoutSeconds=467&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-qtkdl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xIFGKEoyin +++ mktemp ++ local LAST_ERR=/tmp/tmp.QKSB8YDxEE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xIFGKEoyin ++ cat /tmp/tmp.QKSB8YDxEE ++ rm /tmp/tmp.xIFGKEoyin /tmp/tmp.QKSB8YDxEE ++ return 0 + client_pod=pxc-client-67fc4995bb-qtkdl + wait_pod pxc-client-67fc4995bb-qtkdl + local pod=pxc-client-67fc4995bb-qtkdl + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-qtkdl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-qtkdl condition met E0516 21:47:38.068911 12126 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-qtkdl&resourceVersion=1778968056451495000&timeoutSeconds=497&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-qtkdl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PpInAsxIfA +++ mktemp ++ local LAST_ERR=/tmp/tmp.OI2JFzALpk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PpInAsxIfA ++ cat /tmp/tmp.OI2JFzALpk ++ rm /tmp/tmp.PpInAsxIfA /tmp/tmp.OI2JFzALpk ++ return 0 + client_pod=pxc-client-67fc4995bb-qtkdl + wait_pod pxc-client-67fc4995bb-qtkdl + local pod=pxc-client-67fc4995bb-qtkdl + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-qtkdl ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-qtkdl condition met E0516 21:48:16.517526 17146 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-qtkdl&resourceVersion=1778968095090859000&timeoutSeconds=313&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-qtkdl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.itqpKkxwim/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.itqpKkxwim/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.itqpKkxwim/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TG6VVBJqeq +++ mktemp ++ local LAST_ERR=/tmp/tmp.1XCKULfFCv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TG6VVBJqeq ++ cat /tmp/tmp.1XCKULfFCv ++ rm /tmp/tmp.TG6VVBJqeq /tmp/tmp.1XCKULfFCv ++ return 0 + client_pod=pxc-client-67fc4995bb-qtkdl + wait_pod pxc-client-67fc4995bb-qtkdl + local pod=pxc-client-67fc4995bb-qtkdl + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-qtkdl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-qtkdl condition met E0516 21:48:22.731806 17885 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-qtkdl&resourceVersion=1778968101090151000&timeoutSeconds=406&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-qtkdl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.itqpKkxwim/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.itqpKkxwim/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.itqpKkxwim/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hocCN8HtFD +++ mktemp ++ local LAST_ERR=/tmp/tmp.s1AZj9tsRP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hocCN8HtFD ++ cat /tmp/tmp.s1AZj9tsRP ++ rm /tmp/tmp.hocCN8HtFD /tmp/tmp.s1AZj9tsRP ++ return 0 + client_pod=pxc-client-67fc4995bb-qtkdl + wait_pod pxc-client-67fc4995bb-qtkdl + local pod=pxc-client-67fc4995bb-qtkdl + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-qtkdl + local container= + set +o xtrace pod/pxc-client-67fc4995bb-qtkdl condition met E0516 21:48:29.553527 19017 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-qtkdl&resourceVersion=1778968108673645000&timeoutSeconds=574&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-qtkdl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.itqpKkxwim/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.itqpKkxwim/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.itqpKkxwim/select-1.sql + is_keyring_plugin_in_use scheduled-backup + local cluster=scheduled-backup + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + kubectl exec scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + grep -E -o 'early-plugin-load=keyring_\w+.so' + return 1 + sleep 20 + desc 'add backups schedule for pvc storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for pvc storage ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.TfI5U2fIOA +++ mktemp ++ local LAST_ERR=/tmp/tmp.4IXbjfiGBD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TfI5U2fIOA ++ cat /tmp/tmp.4IXbjfiGBD ++ rm /tmp/tmp.TfI5U2fIOA /tmp/tmp.4IXbjfiGBD ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2467-3dc7f023-16-cluster9 --namespace=scheduled-backup-28307 ++ mktemp + local LAST_OUT=/tmp/tmp.IZPdaWon7f ++ mktemp + local LAST_ERR=/tmp/tmp.QFPokgEaQh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2467-3dc7f023-16-cluster9 --namespace=scheduled-backup-28307 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IZPdaWon7f Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2467-3dc7f023-16-cluster9" modified. + cat /tmp/tmp.QFPokgEaQh + rm /tmp/tmp.IZPdaWon7f /tmp/tmp.QFPokgEaQh + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local pvc_name= + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.9kyoIoi4wf + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_ERR=/tmp/tmp.HWQdxHgYKc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9kyoIoi4wf perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.HWQdxHgYKc + rm /tmp/tmp.9kyoIoi4wf /tmp/tmp.HWQdxHgYKc + return 0 + label_node ++ grep -v master ++ awk '{print $1}' ++ kubectl_bin get nodes --no-headers=true ++ head -n1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.TP58UHgiY5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Baeeh13UIe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get nodes --no-headers=true ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TP58UHgiY5 ++ cat /tmp/tmp.Baeeh13UIe ++ rm /tmp/tmp.TP58UHgiY5 /tmp/tmp.Baeeh13UIe ++ return 0 + LABELED_NODE=gke-jen-pxc-2467-3dc7f02-default-pool-6206f5bb-bc0k + kubectl_bin label nodes gke-jen-pxc-2467-3dc7f02-default-pool-6206f5bb-bc0k backupWorker=True --overwrite ++ mktemp + local LAST_OUT=/tmp/tmp.lYJ4yxxEPu ++ mktemp + local LAST_ERR=/tmp/tmp.zm7KxHEhzT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label nodes gke-jen-pxc-2467-3dc7f02-default-pool-6206f5bb-bc0k backupWorker=True --overwrite + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lYJ4yxxEPu node/gke-jen-pxc-2467-3dc7f02-default-pool-6206f5bb-bc0k labeled + cat /tmp/tmp.zm7KxHEhzT + rm /tmp/tmp.lYJ4yxxEPu /tmp/tmp.zm7KxHEhzT + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + kubectl_bin apply -f - + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_OUT=/tmp/tmp.c5Tf3krtXs + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + local LAST_ERR=/tmp/tmp.NfxeUjsin9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.c5Tf3krtXs perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.NfxeUjsin9 + rm /tmp/tmp.c5Tf3krtXs /tmp/tmp.NfxeUjsin9 + return 0 + wait_all_backups ++ get_running_backups_amount ++ wc -l ++ grep -vE ':Succeeded|:Failed' ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uY9wadp2JI +++ mktemp ++ local LAST_ERR=/tmp/tmp.geodjY5Hv7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uY9wadp2JI ++ cat /tmp/tmp.geodjY5Hv7 ++ rm /tmp/tmp.uY9wadp2JI /tmp/tmp.geodjY5Hv7 ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ grep :Failed ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sOqMW8OFGs +++ mktemp ++ local LAST_ERR=/tmp/tmp.4DfeepbRgB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sOqMW8OFGs ++ cat /tmp/tmp.4DfeepbRgB ++ rm /tmp/tmp.sOqMW8OFGs /tmp/tmp.4DfeepbRgB ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-0 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met E0516 21:50:13.264290 32215 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscheduled-backup-pxc-0&resourceVersion=1778968212516479004&timeoutSeconds=384&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met E0516 21:50:19.915554 702 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscheduled-backup-pxc-1&resourceVersion=1778968219530011000&timeoutSeconds=421&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.dkCxuVchoI ++ mktemp + local LAST_ERR=/tmp/tmp.gO5ElHHYUE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dkCxuVchoI NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-2026516214959-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2026516214959-q6fav-20260516214959-57b3e25c Running 35s + cat /tmp/tmp.gO5ElHHYUE + rm /tmp/tmp.dkCxuVchoI /tmp/tmp.gO5ElHHYUE + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.ntznjdMMT4 ++ mktemp + local LAST_ERR=/tmp/tmp.AvYWsM3LDF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ntznjdMMT4 NAME READY STATUS RESTARTS AGE minio-service-5fd5489bdc-c6zbn 1/1 Running 0 8m47s pxc-client-67fc4995bb-qtkdl 2/2 Running 0 7m26s scheduled-backup-proxysql-0 3/3 Running 0 7m23s scheduled-backup-proxysql-1 3/3 Running 0 7m2s scheduled-backup-pxc-0 1/1 Running 0 7m23s scheduled-backup-pxc-1 1/1 Running 0 6m3s scheduled-backup-pxc-2 1/1 Running 0 4m44s xb-cron-scheduled-backup-pvc-2026516214959-q6fav-crqv9 1/1 Running 0 36s + cat /tmp/tmp.AvYWsM3LDF + rm /tmp/tmp.ntznjdMMT4 /tmp/tmp.AvYWsM3LDF + return 0 + sleep 20 ++ get_running_backups_amount ++ grep -vE ':Succeeded|:Failed' ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XsCyCxoqZC +++ mktemp ++ local LAST_ERR=/tmp/tmp.McTOUjVTDs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XsCyCxoqZC ++ cat /tmp/tmp.McTOUjVTDs ++ rm /tmp/tmp.XsCyCxoqZC /tmp/tmp.McTOUjVTDs ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.TVza45Z8t4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.K6t6dkHt4v ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TVza45Z8t4 ++ cat /tmp/tmp.K6t6dkHt4v ++ rm /tmp/tmp.TVza45Z8t4 /tmp/tmp.K6t6dkHt4v ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local pvc_name= ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_OUT=/tmp/tmp.q8MUxak6Js + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.cPebQvBArx + local exit_status=0 + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.q8MUxak6Js perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.cPebQvBArx + rm /tmp/tmp.q8MUxak6Js /tmp/tmp.cPebQvBArx + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + kubectl_bin apply -f - + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + local LAST_OUT=/tmp/tmp.cHkqsLjcUn + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local LAST_ERR=/tmp/tmp.Jxd1rSLL9v + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cHkqsLjcUn perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.Jxd1rSLL9v + rm /tmp/tmp.cHkqsLjcUn /tmp/tmp.Jxd1rSLL9v + return 0 + wait_all_backups ++ get_running_backups_amount ++ wc -l ++ grep -vE ':Succeeded|:Failed' ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.okrPT92gZV +++ mktemp ++ local LAST_ERR=/tmp/tmp.FefXFOtfjR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.okrPT92gZV ++ cat /tmp/tmp.FefXFOtfjR ++ rm /tmp/tmp.okrPT92gZV /tmp/tmp.FefXFOtfjR ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.a8gdC9Byky +++ mktemp ++ local LAST_ERR=/tmp/tmp.46HG60VPIU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.a8gdC9Byky ++ cat /tmp/tmp.46HG60VPIU ++ rm /tmp/tmp.a8gdC9Byky /tmp/tmp.46HG60VPIU ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-0 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met E0516 21:52:21.590628 18358 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscheduled-backup-pxc-0&resourceVersion=1778968339915271000&timeoutSeconds=457&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met E0516 21:52:47.054769 22111 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscheduled-backup-pxc-2&resourceVersion=1778968365610039000&timeoutSeconds=452&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.NcXmnkfyNQ ++ mktemp + local LAST_ERR=/tmp/tmp.edelzCXgvv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NcXmnkfyNQ NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-202651621525-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2026-05-16-21:52:05-full Succeeded 15s 50s cron-scheduled-backup-pvc-2026516214959-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2026516214959-q6fav-20260516214959-57b3e25c Succeeded 2m6s 2m56s + cat /tmp/tmp.edelzCXgvv + rm /tmp/tmp.NcXmnkfyNQ /tmp/tmp.edelzCXgvv + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.7As5YP0t7w ++ mktemp + local LAST_ERR=/tmp/tmp.wVdXC4rP54 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7As5YP0t7w NAME READY STATUS RESTARTS AGE minio-service-5fd5489bdc-c6zbn 1/1 Running 0 11m pxc-client-67fc4995bb-qtkdl 2/2 Running 0 9m48s scheduled-backup-proxysql-0 3/3 Running 0 9m45s scheduled-backup-proxysql-1 3/3 Running 0 9m24s scheduled-backup-pxc-0 1/1 Running 0 9m45s scheduled-backup-pxc-1 1/1 Running 0 8m25s scheduled-backup-pxc-2 1/1 Running 0 7m6s xb-cron-scheduled-backup-aws-s3-202651621525-q6fav-cc9ml 0/1 Completed 0 52s xb-cron-scheduled-backup-pvc-2026516214959-q6fav-crqv9 0/1 Completed 0 2m58s + cat /tmp/tmp.wVdXC4rP54 + rm /tmp/tmp.7As5YP0t7w /tmp/tmp.wVdXC4rP54 + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VUHCAaSU05 +++ mktemp ++ local LAST_ERR=/tmp/tmp.0w2ipdgmv6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VUHCAaSU05 ++ cat /tmp/tmp.0w2ipdgmv6 ++ rm /tmp/tmp.VUHCAaSU05 /tmp/tmp.0w2ipdgmv6 ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.3M2mRN41TH +++ mktemp ++ local LAST_ERR=/tmp/tmp.0dNyFj66wq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3M2mRN41TH ++ cat /tmp/tmp.0dNyFj66wq ++ rm /tmp/tmp.3M2mRN41TH /tmp/tmp.0dNyFj66wq ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for minio storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for minio storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.2Ltdh4gcmx ++ mktemp + local LAST_ERR=/tmp/tmp.V4mEWTodWw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2Ltdh4gcmx perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.V4mEWTodWw + rm /tmp/tmp.2Ltdh4gcmx /tmp/tmp.V4mEWTodWw + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + kubectl_bin apply -f - + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' ++ mktemp + local LAST_OUT=/tmp/tmp.dFH7ezW4O8 + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + local LAST_ERR=/tmp/tmp.sRzatHZuKq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dFH7ezW4O8 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.sRzatHZuKq + rm /tmp/tmp.dFH7ezW4O8 /tmp/tmp.sRzatHZuKq + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v7JtLdM7Ko +++ mktemp ++ local LAST_ERR=/tmp/tmp.a7utG0d7OC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.v7JtLdM7Ko ++ cat /tmp/tmp.a7utG0d7OC ++ rm /tmp/tmp.v7JtLdM7Ko /tmp/tmp.a7utG0d7OC ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ grep :Failed ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BYQfUEE4zK +++ mktemp ++ local LAST_ERR=/tmp/tmp.QkF74a4lmG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BYQfUEE4zK ++ cat /tmp/tmp.QkF74a4lmG ++ rm /tmp/tmp.BYQfUEE4zK /tmp/tmp.QkF74a4lmG ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-1 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-2 + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.RQV06mWXJC ++ mktemp + local LAST_ERR=/tmp/tmp.4vBPhQrrbE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RQV06mWXJC NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-202651621525-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2026-05-16-21:52:05-full Succeeded 2m24s 2m59s cron-scheduled-backup-minio-2026516215424-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2026-05-16-21:54:24-full Succeeded 8s 40s cron-scheduled-backup-pvc-2026516214959-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2026516214959-q6fav-20260516214959-57b3e25c Succeeded 4m15s 5m5s + cat /tmp/tmp.4vBPhQrrbE + rm /tmp/tmp.RQV06mWXJC /tmp/tmp.4vBPhQrrbE + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.OHUGs5viy8 ++ mktemp + local LAST_ERR=/tmp/tmp.PqNGiJdXEh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OHUGs5viy8 NAME READY STATUS RESTARTS AGE minio-service-5fd5489bdc-c6zbn 1/1 Running 0 13m pxc-client-67fc4995bb-qtkdl 2/2 Running 0 11m scheduled-backup-proxysql-0 3/3 Running 0 11m scheduled-backup-proxysql-1 3/3 Running 0 11m scheduled-backup-pxc-0 1/1 Running 0 11m scheduled-backup-pxc-1 1/1 Running 0 10m scheduled-backup-pxc-2 1/1 Running 0 9m14s xb-cron-scheduled-backup-aws-s3-202651621525-q6fav-cc9ml 0/1 Completed 0 3m xb-cron-scheduled-backup-minio-2026516215424-q6fav-vzf2s 0/1 Completed 0 41s xb-cron-scheduled-backup-pvc-2026516214959-q6fav-crqv9 0/1 Completed 0 5m6s + cat /tmp/tmp.PqNGiJdXEh + rm /tmp/tmp.OHUGs5viy8 /tmp/tmp.PqNGiJdXEh + return 0 + sleep 20 ++ get_running_backups_amount ++ wc -l ++ grep -vE ':Succeeded|:Failed' ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JhH3mTvV3f +++ mktemp ++ local LAST_ERR=/tmp/tmp.x2DDDFRozK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JhH3mTvV3f ++ cat /tmp/tmp.x2DDDFRozK ++ rm /tmp/tmp.JhH3mTvV3f /tmp/tmp.x2DDDFRozK ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ grep :Failed ++ local LAST_OUT=/tmp/tmp.k1pKaPnVC0 ++ wc -l +++ mktemp ++ local LAST_ERR=/tmp/tmp.kU9zTmO73Z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.k1pKaPnVC0 ++ cat /tmp/tmp.kU9zTmO73Z ++ rm /tmp/tmp.k1pKaPnVC0 /tmp/tmp.kU9zTmO73Z ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for gcs storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for gcs storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + kubectl_bin apply -f - + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.FifusFrQRb + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_ERR=/tmp/tmp.2wydjXr4FK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FifusFrQRb perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.2wydjXr4FK + rm /tmp/tmp.FifusFrQRb /tmp/tmp.2wydjXr4FK + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + local LAST_OUT=/tmp/tmp.ziYKcGEwra + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_ERR=/tmp/tmp.vbifswae55 + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ziYKcGEwra perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.vbifswae55 + rm /tmp/tmp.ziYKcGEwra /tmp/tmp.vbifswae55 + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.9wRE2qjFid +++ mktemp ++ local LAST_ERR=/tmp/tmp.CldxSL2QHh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9wRE2qjFid ++ cat /tmp/tmp.CldxSL2QHh ++ rm /tmp/tmp.9wRE2qjFid /tmp/tmp.CldxSL2QHh ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep :Failed ++ local LAST_OUT=/tmp/tmp.b8zlYko5r2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.felQ4UJZdE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.b8zlYko5r2 ++ cat /tmp/tmp.felQ4UJZdE ++ rm /tmp/tmp.b8zlYko5r2 /tmp/tmp.felQ4UJZdE ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met E0516 21:56:40.050234 19478 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscheduled-backup-pxc-0&resourceVersion=1778968595379167002&timeoutSeconds=554&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met E0516 21:56:44.551721 19960 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscheduled-backup-pxc-1&resourceVersion=1778968600380049000&timeoutSeconds=308&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met E0516 21:56:48.211280 20355 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscheduled-backup-pxc-2&resourceVersion=1778968605380201000&timeoutSeconds=353&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.hjaAL9TANQ ++ mktemp + local LAST_ERR=/tmp/tmp.19ToK6Wxbh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hjaAL9TANQ NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-202651621525-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2026-05-16-21:52:05-full Succeeded 4m12s 4m47s cron-scheduled-backup-gcp-cs-2026516215632-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2026-05-16-21:56:32-full Running 20s cron-scheduled-backup-minio-2026516215424-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2026-05-16-21:54:24-full Succeeded 116s 2m28s cron-scheduled-backup-pvc-2026516214959-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2026516214959-q6fav-20260516214959-57b3e25c Succeeded 6m3s 6m53s + cat /tmp/tmp.19ToK6Wxbh + rm /tmp/tmp.hjaAL9TANQ /tmp/tmp.19ToK6Wxbh + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.uTeg8lYMPY ++ mktemp + local LAST_ERR=/tmp/tmp.QjFobKCxp6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uTeg8lYMPY NAME READY STATUS RESTARTS AGE minio-service-5fd5489bdc-c6zbn 1/1 Running 0 15m pxc-client-67fc4995bb-qtkdl 2/2 Running 0 13m scheduled-backup-proxysql-0 3/3 Running 0 13m scheduled-backup-proxysql-1 3/3 Running 0 13m scheduled-backup-pxc-0 1/1 Running 0 13m scheduled-backup-pxc-1 1/1 Running 0 12m scheduled-backup-pxc-2 1/1 Running 0 11m xb-cron-scheduled-backup-aws-s3-202651621525-q6fav-cc9ml 0/1 Completed 0 4m48s xb-cron-scheduled-backup-gcp-cs-2026516215632-q6fav-6rzqz 1/1 Running 0 21s xb-cron-scheduled-backup-minio-2026516215424-q6fav-vzf2s 0/1 Completed 0 2m29s xb-cron-scheduled-backup-pvc-2026516214959-q6fav-crqv9 0/1 Completed 0 6m54s + cat /tmp/tmp.QjFobKCxp6 + rm /tmp/tmp.uTeg8lYMPY /tmp/tmp.QjFobKCxp6 + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.1h7bmW0ycF +++ mktemp ++ local LAST_ERR=/tmp/tmp.CXlh5v1mDm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1h7bmW0ycF ++ cat /tmp/tmp.CXlh5v1mDm ++ rm /tmp/tmp.1h7bmW0ycF /tmp/tmp.CXlh5v1mDm ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.PcvPR3xg37 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Fp7Y2szDVr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PcvPR3xg37 ++ cat /tmp/tmp.Fp7Y2szDVr ++ rm /tmp/tmp.PcvPR3xg37 /tmp/tmp.Fp7Y2szDVr ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for azure storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for azure storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_OUT=/tmp/tmp.7PnctCbAyq + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + local LAST_ERR=/tmp/tmp.qVXpYdrSBx + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7PnctCbAyq perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.qVXpYdrSBx + rm /tmp/tmp.7PnctCbAyq /tmp/tmp.qVXpYdrSBx + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_OUT=/tmp/tmp.YO8ivsjOsj + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.sgvgCA6tj9 + local exit_status=0 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YO8ivsjOsj perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.sgvgCA6tj9 + rm /tmp/tmp.YO8ivsjOsj /tmp/tmp.sgvgCA6tj9 + return 0 + wait_all_backups ++ get_running_backups_amount ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VRZgik8D0j +++ mktemp ++ local LAST_ERR=/tmp/tmp.nAbQE6seiA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VRZgik8D0j ++ cat /tmp/tmp.nAbQE6seiA ++ rm /tmp/tmp.VRZgik8D0j /tmp/tmp.nAbQE6seiA ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.j1mIKAxZuf +++ mktemp ++ local LAST_ERR=/tmp/tmp.UzYKZUpKQ6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.j1mIKAxZuf ++ cat /tmp/tmp.UzYKZUpKQ6 ++ rm /tmp/tmp.j1mIKAxZuf /tmp/tmp.UzYKZUpKQ6 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-0 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met E0516 21:58:28.657754 30578 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscheduled-backup-pxc-0&resourceVersion=1778968703601135017&timeoutSeconds=382&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met E0516 21:58:35.220830 31292 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscheduled-backup-pxc-1&resourceVersion=1778968713605078000&timeoutSeconds=401&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met E0516 21:58:41.590836 31953 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscheduled-backup-pxc-2&resourceVersion=1778968718605482000&timeoutSeconds=513&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.2Idz72paLK ++ mktemp + local LAST_ERR=/tmp/tmp.dpIS6RwXJA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2Idz72paLK NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-202651621525-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2026-05-16-21:52:05-full Succeeded 6m6s 6m41s cron-scheduled-backup-azure-blob-2026516215818-q6fav scheduled-backup azure-blob azure://operator-testing/scheduled-backup-2026-05-16-21:58:18-full Running 28s cron-scheduled-backup-gcp-cs-2026516215632-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2026-05-16-21:56:32-full Succeeded 103s 2m14s cron-scheduled-backup-minio-2026516215424-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2026-05-16-21:54:24-full Succeeded 3m50s 4m22s cron-scheduled-backup-pvc-2026516214959-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2026516214959-q6fav-20260516214959-57b3e25c Succeeded 7m57s 8m47s + cat /tmp/tmp.dpIS6RwXJA + rm /tmp/tmp.2Idz72paLK /tmp/tmp.dpIS6RwXJA + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.x3tUBLL5Mt ++ mktemp + local LAST_ERR=/tmp/tmp.wD1d3Pf94S + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.x3tUBLL5Mt NAME READY STATUS RESTARTS AGE minio-service-5fd5489bdc-c6zbn 1/1 Running 0 16m pxc-client-67fc4995bb-qtkdl 2/2 Running 0 15m scheduled-backup-proxysql-0 3/3 Running 0 15m scheduled-backup-proxysql-1 3/3 Running 0 15m scheduled-backup-pxc-0 1/1 Running 0 15m scheduled-backup-pxc-1 1/1 Running 0 14m scheduled-backup-pxc-2 1/1 Running 0 12m xb-cron-scheduled-backup-aws-s3-202651621525-q6fav-cc9ml 0/1 Completed 0 6m41s xb-cron-scheduled-backup-azure-blob-2026516215818-q6fav-z29pm 0/1 Completed 0 28s xb-cron-scheduled-backup-gcp-cs-2026516215632-q6fav-6rzqz 0/1 Completed 0 2m14s xb-cron-scheduled-backup-minio-2026516215424-q6fav-vzf2s 0/1 Completed 0 4m22s xb-cron-scheduled-backup-pvc-2026516214959-q6fav-crqv9 0/1 Completed 0 8m47s + cat /tmp/tmp.wD1d3Pf94S + rm /tmp/tmp.x3tUBLL5Mt /tmp/tmp.wD1d3Pf94S + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.QqEeiKXPwJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.cZpUNB2bMw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QqEeiKXPwJ ++ cat /tmp/tmp.cZpUNB2bMw ++ rm /tmp/tmp.QqEeiKXPwJ /tmp/tmp.cZpUNB2bMw ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ grep :Failed ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1zcdUD5Tdh +++ mktemp ++ local LAST_ERR=/tmp/tmp.e06r9ySHep ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1zcdUD5Tdh ++ cat /tmp/tmp.e06r9ySHep ++ rm /tmp/tmp.1zcdUD5Tdh /tmp/tmp.e06r9ySHep ++ return 0 + [[ 0 -gt 0 ]] ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ head -n1 ++ cut -d: -f1 ++ grep pvc ++ grep Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.ctEqI0odnf +++ mktemp ++ local LAST_ERR=/tmp/tmp.sirhIt3kGC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ctEqI0odnf ++ cat /tmp/tmp.sirhIt3kGC ++ rm /tmp/tmp.ctEqI0odnf /tmp/tmp.sirhIt3kGC ++ return 0 + FIRST_PVC_BACKUP=cron-scheduled-backup-pvc-2026516214959-q6fav ++ kubectl_bin get jobs ++ awk '{print $1}' ++ grep cron-scheduled-backup-pvc-2026516214959-q6fav +++ mktemp ++ local LAST_OUT=/tmp/tmp.r7YvgT6Rl5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FFqEwlkRY8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.r7YvgT6Rl5 ++ cat /tmp/tmp.FFqEwlkRY8 ++ rm /tmp/tmp.r7YvgT6Rl5 /tmp/tmp.FFqEwlkRY8 ++ return 0 + JOB_PVC_BACKUP=xb-cron-scheduled-backup-pvc-2026516214959-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-pvc-2026516214959 +++ mktemp ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.BGsBIfiiWT +++ mktemp ++ local LAST_ERR=/tmp/tmp.XATZK6gvwX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BGsBIfiiWT ++ cat /tmp/tmp.XATZK6gvwX ++ rm /tmp/tmp.BGsBIfiiWT /tmp/tmp.XATZK6gvwX ++ return 0 + POD_PVC_BACKUP=xb-cron-scheduled-backup-pvc-2026516214959-q6fav-crqv9 ++ grep minio ++ head -n1 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ cut -d: -f1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.M5Ev8zC9Aa +++ mktemp ++ local LAST_ERR=/tmp/tmp.L7LESaUfaP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.M5Ev8zC9Aa ++ cat /tmp/tmp.L7LESaUfaP ++ rm /tmp/tmp.M5Ev8zC9Aa /tmp/tmp.L7LESaUfaP ++ return 0 + FIRST_MINIO_BACKUP=cron-scheduled-backup-minio-2026516215424-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-minio-2026516215424-q6fav +++ mktemp ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.unqTgd4qFG +++ mktemp ++ local LAST_ERR=/tmp/tmp.cGD8R6OOeA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.unqTgd4qFG ++ cat /tmp/tmp.cGD8R6OOeA ++ rm /tmp/tmp.unqTgd4qFG /tmp/tmp.cGD8R6OOeA ++ return 0 + JOB_MINIO_BACKUP=xb-cron-scheduled-backup-minio-2026516215424-q6fav ++ kubectl_bin get pods ++ awk '{print $1}' +++ mktemp ++ grep xb-cron-scheduled-backup-minio-2026516215424 ++ local LAST_OUT=/tmp/tmp.5QJ2tyPa4K +++ mktemp ++ local LAST_ERR=/tmp/tmp.NFNCVhy1BX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5QJ2tyPa4K ++ cat /tmp/tmp.NFNCVhy1BX ++ rm /tmp/tmp.5QJ2tyPa4K /tmp/tmp.NFNCVhy1BX ++ return 0 + POD_MINIO_BACKUP=xb-cron-scheduled-backup-minio-2026516215424-q6fav-vzf2s + '[' -z '' ']' + command -v aws + echo 'AWS CLI is installed' AWS CLI is installed + command -v gsutil + echo 'gutil is installed' gutil is installed + command -v az + echo 'Azure CLI is installed' Azure CLI is installed ++ grep aws ++ grep Succeeded ++ head -n1 ++ cut -d: -f1 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yQw34Sjq1q +++ mktemp ++ local LAST_ERR=/tmp/tmp.5kveSeAsvp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yQw34Sjq1q ++ cat /tmp/tmp.5kveSeAsvp ++ rm /tmp/tmp.yQw34Sjq1q /tmp/tmp.5kveSeAsvp ++ return 0 + FIRST_AWS_BACKUP=cron-scheduled-backup-aws-s3-202651621525-q6fav ++ awk '{print $1}' ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-aws-s3-202651621525-q6fav +++ mktemp ++ local LAST_OUT=/tmp/tmp.HmkkiXatA8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.EHemBsm63t ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HmkkiXatA8 ++ cat /tmp/tmp.EHemBsm63t ++ rm /tmp/tmp.HmkkiXatA8 /tmp/tmp.EHemBsm63t ++ return 0 + JOB_AWS_BACKUP=xb-cron-scheduled-backup-aws-s3-202651621525-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-aws-s3-202651621525 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sXLo1HTxWC +++ mktemp ++ local LAST_ERR=/tmp/tmp.k0npm6eYdC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sXLo1HTxWC ++ cat /tmp/tmp.k0npm6eYdC ++ rm /tmp/tmp.sXLo1HTxWC /tmp/tmp.k0npm6eYdC ++ return 0 + POD_AWS_BACKUP=xb-cron-scheduled-backup-aws-s3-202651621525-q6fav-cc9ml ++ cut -d: -f1 ++ head -n1 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ grep gcp +++ mktemp ++ local LAST_OUT=/tmp/tmp.usoTvyO1vo +++ mktemp ++ local LAST_ERR=/tmp/tmp.IPgCi5Nqy6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.usoTvyO1vo ++ cat /tmp/tmp.IPgCi5Nqy6 ++ rm /tmp/tmp.usoTvyO1vo /tmp/tmp.IPgCi5Nqy6 ++ return 0 + FIRST_GCP_BACKUP=cron-scheduled-backup-gcp-cs-2026516215632-q6fav ++ awk '{print $1}' ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-gcp-cs-2026516215632-q6fav +++ mktemp ++ local LAST_OUT=/tmp/tmp.kGJLGPgD6l +++ mktemp ++ local LAST_ERR=/tmp/tmp.n9VcFZ6N5C ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kGJLGPgD6l ++ cat /tmp/tmp.n9VcFZ6N5C ++ rm /tmp/tmp.kGJLGPgD6l /tmp/tmp.n9VcFZ6N5C ++ return 0 + JOB_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-2026516215632-q6fav ++ awk '{print $1}' ++ grep xb-cron-scheduled-backup-gcp-cs-2026516215632 ++ kubectl_bin get pods +++ mktemp ++ local LAST_OUT=/tmp/tmp.YpDEu4VR2N +++ mktemp ++ local LAST_ERR=/tmp/tmp.8t04Ts1h9L ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YpDEu4VR2N ++ cat /tmp/tmp.8t04Ts1h9L ++ rm /tmp/tmp.YpDEu4VR2N /tmp/tmp.8t04Ts1h9L ++ return 0 + POD_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-2026516215632-q6fav-6rzqz ++ cut -d: -f1 ++ head -n1 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' +++ mktemp ++ grep Succeeded ++ grep azure ++ local LAST_OUT=/tmp/tmp.Ax5lnIOzN8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.u7sQx1LygZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ax5lnIOzN8 ++ cat /tmp/tmp.u7sQx1LygZ ++ rm /tmp/tmp.Ax5lnIOzN8 /tmp/tmp.u7sQx1LygZ ++ return 0 + FIRST_AZURE_BACKUP=cron-scheduled-backup-azure-blob-2026516215818-q6fav ++ kubectl_bin get jobs +++ mktemp ++ grep cron-scheduled-backup-azure-blob-2026516215818-q6fav ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.7O83ZBYYXX +++ mktemp ++ local LAST_ERR=/tmp/tmp.uRDHSW9sTa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7O83ZBYYXX ++ cat /tmp/tmp.uRDHSW9sTa ++ rm /tmp/tmp.7O83ZBYYXX /tmp/tmp.uRDHSW9sTa ++ return 0 + JOB_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-2026516215818-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-azure-blob-2026516215818 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4kqw2lp6fk +++ mktemp ++ local LAST_ERR=/tmp/tmp.5EGbQVVLfl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4kqw2lp6fk ++ cat /tmp/tmp.5EGbQVVLfl ++ rm /tmp/tmp.4kqw2lp6fk /tmp/tmp.5EGbQVVLfl ++ return 0 + POD_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-2026516215818-q6fav-z29pm ++ sed -e 's/.json$//' ++ kubectl_bin get pxc-backup cron-scheduled-backup-aws-s3-202651621525-q6fav -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NoIsGxNPHs +++ mktemp ++ cut -c 6- ++ local LAST_ERR=/tmp/tmp.vsvtMYLa6P ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-aws-s3-202651621525-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NoIsGxNPHs ++ cat /tmp/tmp.vsvtMYLa6P ++ rm /tmp/tmp.NoIsGxNPHs /tmp/tmp.vsvtMYLa6P ++ return 0 + BACKUP_DEST_AWS=operator-testing/scheduled-backup-2026-05-16-21:52:05-full ++ sed -e 's/.json$//' ++ cut -c 6- ++ kubectl_bin get pxc-backup cron-scheduled-backup-gcp-cs-2026516215632-q6fav -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sOGlKAs3Ta +++ mktemp ++ local LAST_ERR=/tmp/tmp.COHftQiD0f ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-gcp-cs-2026516215632-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sOGlKAs3Ta ++ cat /tmp/tmp.COHftQiD0f ++ rm /tmp/tmp.sOGlKAs3Ta /tmp/tmp.COHftQiD0f ++ return 0 + BACKUP_DEST_GCP=operator-testing/scheduled-backup-2026-05-16-21:56:32-full ++ sed -e 's/.json$//' ++ cut -c 9- ++ kubectl_bin get pxc-backup cron-scheduled-backup-azure-blob-2026516215818-q6fav -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ofyRCnoNDT +++ mktemp ++ local LAST_ERR=/tmp/tmp.H9X7eIU911 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-azure-blob-2026516215818-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ofyRCnoNDT ++ cat /tmp/tmp.H9X7eIU911 ++ rm /tmp/tmp.ofyRCnoNDT /tmp/tmp.H9X7eIU911 ++ return 0 + BACKUP_DEST_AZURE=operator-testing/scheduled-backup-2026-05-16-21:58:18-full + desc 'Check backup existence' + set +o xtrace ----------------------------------------------------------------------------------- Check backup existence ----------------------------------------------------------------------------------- + setup_aws_credentials + local secret_name=aws-s3-secret + [[ -n AKIARXP3OARBMEQ6FSEW ]] + [[ -n gljUukL3/TuQnzrtJUgiURH6vqjt+SEo89jGS2Z/ ]] + echo 'AWS credentials already set in environment' AWS credentials already set in environment + return 0 + check_backup_existence_aws operator-testing/scheduled-backup-2026-05-16-21:52:05-full .sst_info/sst_info.00000000000000000000 ++ cut -d/ -f1 ++ echo operator-testing/scheduled-backup-2026-05-16-21:52:05-full + bucket=operator-testing ++ echo operator-testing/scheduled-backup-2026-05-16-21:52:05-full ++ cut -d/ -f2- + key_prefix=scheduled-backup-2026-05-16-21:52:05-full + key=.sst_info/sst_info.00000000000000000000 + storage_name=aws-s3 + retry=0 + aws s3api head-object --bucket operator-testing --key scheduled-backup-2026-05-16-21:52:05-full.sst_info/sst_info.00000000000000000000 + echo 'Backup scheduled-backup-2026-05-16-21:52:05-full.sst_info/sst_info.00000000000000000000 found in bucket operator-testing in aws-s3' Backup scheduled-backup-2026-05-16-21:52:05-full.sst_info/sst_info.00000000000000000000 found in bucket operator-testing in aws-s3 + setup_gcs_credentials + local secret_name=gcp-cs-secret + gsutil ls + echo 'Setting up GCS credentials from K8s secret: gcp-cs-secret' Setting up GCS credentials from K8s secret: gcp-cs-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'GCS credentials configured successfully' GCS credentials configured successfully + check_backup_existence_gcs operator-testing/scheduled-backup-2026-05-16-21:56:32-full + backup_dest_gcp=operator-testing/scheduled-backup-2026-05-16-21:56:32-full + storage_name=gcp-cs + retry=0 + key=.sst_info/sst_info.00000000000000000000 + gcs_path=gs://operator-testing/scheduled-backup-2026-05-16-21:56:32-full.sst_info/sst_info.00000000000000000000 + gsutil ls gs://operator-testing/scheduled-backup-2026-05-16-21:56:32-full.sst_info/sst_info.00000000000000000000 + echo 'Backup found in gcp-cs: gs://operator-testing/scheduled-backup-2026-05-16-21:56:32-full.sst_info/sst_info.00000000000000000000' Backup found in gcp-cs: gs://operator-testing/scheduled-backup-2026-05-16-21:56:32-full.sst_info/sst_info.00000000000000000000 + setup_azure_credentials + local secret_name=azure-secret + echo 'Setting up Azure credentials from K8s secret: azure-secret' Setting up Azure credentials from K8s secret: azure-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'Azure credentials configured successfully' Azure credentials configured successfully + check_backup_existence_azure operator-testing/scheduled-backup-2026-05-16-21:58:18-full .sst_info/sst_info.00000000000000000000 ++ cut -d/ -f1 ++ echo operator-testing/scheduled-backup-2026-05-16-21:58:18-full + container=operator-testing ++ echo operator-testing/scheduled-backup-2026-05-16-21:58:18-full ++ cut -d/ -f2- + blob_prefix=scheduled-backup-2026-05-16-21:58:18-full + blob=.sst_info/sst_info.00000000000000000000 + storage_name=azure-blob + retry=0 + blob_path=scheduled-backup-2026-05-16-21:58:18-full.sst_info/sst_info.00000000000000000000 + az storage blob show --container-name operator-testing --name scheduled-backup-2026-05-16-21:58:18-full.sst_info/sst_info.00000000000000000000 + echo 'Backup scheduled-backup-2026-05-16-21:58:18-full.sst_info/sst_info.00000000000000000000 found in container operator-testing in azure-blob' Backup scheduled-backup-2026-05-16-21:58:18-full.sst_info/sst_info.00000000000000000000 found in container operator-testing in azure-blob + desc 'Check that KEEP option saves correct backup'\''s amount (1 for our settings)' + set +o xtrace ----------------------------------------------------------------------------------- Check that KEEP option saves correct backup's amount (1 for our settings) ----------------------------------------------------------------------------------- + desc 'add backups schedule for gcs storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for gcs storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local pvc_name= + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + local LAST_OUT=/tmp/tmp.SFfy8tYbAD ++ mktemp + local LAST_ERR=/tmp/tmp.kiWjseAT43 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SFfy8tYbAD perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.kiWjseAT43 + rm /tmp/tmp.SFfy8tYbAD /tmp/tmp.kiWjseAT43 + return 0 + sleep 50 + wait_num_backups gcp-cs 2 + local storage=gcp-cs + local num=2 + local timer=0 + echo 'Starting to check number 2 of backups for gcp-cs' Starting to check number 2 of backups for gcp-cs ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ wc -l ++ grep gcp-cs ++ grep -E :Succeeded ++ local LAST_OUT=/tmp/tmp.AXmhHVTEIR +++ mktemp ++ local LAST_ERR=/tmp/tmp.SIe9s4FRvD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AXmhHVTEIR ++ cat /tmp/tmp.SIe9s4FRvD ++ rm /tmp/tmp.AXmhHVTEIR /tmp/tmp.SIe9s4FRvD ++ return 0 + [[ 1 -ne 2 ]] + [[ 0 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.R86eq2Pnjn +++ mktemp ++ local LAST_ERR=/tmp/tmp.sWRmYj86FO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.R86eq2Pnjn ++ cat /tmp/tmp.sWRmYj86FO ++ rm /tmp/tmp.R86eq2Pnjn /tmp/tmp.sWRmYj86FO ++ return 0 + [[ 1 -ne 2 ]] + [[ 1 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -E :Succeeded ++ grep gcp-cs +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.zCKVbPy6uj +++ mktemp ++ local LAST_ERR=/tmp/tmp.VQquPXIgrf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zCKVbPy6uj ++ cat /tmp/tmp.VQquPXIgrf ++ rm /tmp/tmp.zCKVbPy6uj /tmp/tmp.VQquPXIgrf ++ return 0 + [[ 1 -ne 2 ]] + [[ 2 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ wc -l ++ grep gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ONGn9n1z8K ++ grep -E :Succeeded +++ mktemp ++ local LAST_ERR=/tmp/tmp.JLVyE1ts9J ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ONGn9n1z8K ++ cat /tmp/tmp.JLVyE1ts9J ++ rm /tmp/tmp.ONGn9n1z8K /tmp/tmp.JLVyE1ts9J ++ return 0 + [[ 1 -ne 2 ]] + [[ 3 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ grep -E :Succeeded ++ grep gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.yFnv1nN18L +++ mktemp ++ local LAST_ERR=/tmp/tmp.rSZEPxXumH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yFnv1nN18L ++ cat /tmp/tmp.rSZEPxXumH ++ rm /tmp/tmp.yFnv1nN18L /tmp/tmp.rSZEPxXumH ++ return 0 + [[ 1 -ne 2 ]] + [[ 4 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ grep gcp-cs ++ local LAST_OUT=/tmp/tmp.ZrLUICQv5M +++ mktemp ++ local LAST_ERR=/tmp/tmp.D0W17RWklA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZrLUICQv5M ++ cat /tmp/tmp.D0W17RWklA ++ rm /tmp/tmp.ZrLUICQv5M /tmp/tmp.D0W17RWklA ++ return 0 + [[ 1 -ne 2 ]] + [[ 5 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ grep -E :Succeeded ++ grep gcp-cs ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xlemTFxRKC +++ mktemp ++ local LAST_ERR=/tmp/tmp.XgyXgYyT0j ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xlemTFxRKC ++ cat /tmp/tmp.XgyXgYyT0j ++ rm /tmp/tmp.xlemTFxRKC /tmp/tmp.XgyXgYyT0j ++ return 0 + [[ 1 -ne 2 ]] + [[ 6 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ grep gcp-cs ++ grep -E :Succeeded ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q3QeUBUTiA +++ mktemp ++ local LAST_ERR=/tmp/tmp.tNeCT1I9mj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.q3QeUBUTiA ++ cat /tmp/tmp.tNeCT1I9mj ++ rm /tmp/tmp.q3QeUBUTiA /tmp/tmp.tNeCT1I9mj ++ return 0 + [[ 1 -ne 2 ]] + [[ 7 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ grep gcp-cs ++ grep -E :Succeeded ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V5oSEaCKsO +++ mktemp ++ wc -l ++ local LAST_ERR=/tmp/tmp.UVBBmk0GoD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.V5oSEaCKsO ++ cat /tmp/tmp.UVBBmk0GoD ++ rm /tmp/tmp.V5oSEaCKsO /tmp/tmp.UVBBmk0GoD ++ return 0 + [[ 1 -ne 2 ]] + [[ 8 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ grep gcp-cs ++ grep -E :Succeeded ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9TjdyarYRW ++ wc -l +++ mktemp ++ local LAST_ERR=/tmp/tmp.ID7aNUVUII ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9TjdyarYRW ++ cat /tmp/tmp.ID7aNUVUII ++ rm /tmp/tmp.9TjdyarYRW /tmp/tmp.ID7aNUVUII ++ return 0 + [[ 1 -ne 2 ]] + [[ 9 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ grep -E :Succeeded ++ grep gcp-cs ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oXzbMhf5NJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.IQC6kCH6I8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oXzbMhf5NJ ++ cat /tmp/tmp.IQC6kCH6I8 ++ rm /tmp/tmp.oXzbMhf5NJ /tmp/tmp.IQC6kCH6I8 ++ return 0 + [[ 1 -ne 2 ]] + [[ 10 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs +++ mktemp ++ grep -E :Succeeded ++ wc -l ++ local LAST_OUT=/tmp/tmp.SkXaK08590 +++ mktemp ++ local LAST_ERR=/tmp/tmp.jYs19HQAN3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SkXaK08590 ++ cat /tmp/tmp.jYs19HQAN3 ++ rm /tmp/tmp.SkXaK08590 /tmp/tmp.jYs19HQAN3 ++ return 0 + [[ 1 -ne 2 ]] + [[ 11 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep gcp-cs ++ grep -E :Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.OTUMMYmmYi +++ mktemp ++ local LAST_ERR=/tmp/tmp.nu6NK05ELw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OTUMMYmmYi ++ cat /tmp/tmp.nu6NK05ELw ++ rm /tmp/tmp.OTUMMYmmYi /tmp/tmp.nu6NK05ELw ++ return 0 + [[ 1 -ne 2 ]] + [[ 12 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ wc -l ++ grep -E :Succeeded ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs +++ mktemp ++ local LAST_OUT=/tmp/tmp.iw6vm2KHO5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.7Xw1J3q6f2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iw6vm2KHO5 ++ cat /tmp/tmp.7Xw1J3q6f2 ++ rm /tmp/tmp.iw6vm2KHO5 /tmp/tmp.7Xw1J3q6f2 ++ return 0 + [[ 2 -ne 2 ]] + echo 'Finished to check number 2 of backups for gcp-cs. Timer: 13' Finished to check number 2 of backups for gcp-cs. Timer: 13 + echo 'Patch backup for yearly backup for gcs storage' Patch backup for yearly backup for gcs storage + kubectl patch pxc scheduled-backup --type=merge -p '{"spec":{"backup":{"schedule":[{"name":"each-min-gcp-cs","storageName":"gcp-cs","keep":1,"schedule":"0 0 1 * *"}]}}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched + wait_num_backups gcp-cs 1 + local storage=gcp-cs + local num=1 + local timer=0 + echo 'Starting to check number 1 of backups for gcp-cs' Starting to check number 1 of backups for gcp-cs ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ wc -l ++ grep -E :Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pmo5p28dJk +++ mktemp ++ local LAST_ERR=/tmp/tmp.5K9o2G2Rz7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Pmo5p28dJk ++ cat /tmp/tmp.5K9o2G2Rz7 ++ rm /tmp/tmp.Pmo5p28dJk /tmp/tmp.5K9o2G2Rz7 ++ return 0 + [[ 2 -ne 1 ]] + [[ 0 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -E :Succeeded ++ grep gcp-cs ++ local LAST_OUT=/tmp/tmp.2J2Y1MijJK +++ mktemp ++ local LAST_ERR=/tmp/tmp.Y7Qmrp694k ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2J2Y1MijJK ++ cat /tmp/tmp.Y7Qmrp694k ++ rm /tmp/tmp.2J2Y1MijJK /tmp/tmp.Y7Qmrp694k ++ return 0 + [[ 2 -ne 1 ]] + [[ 1 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep gcp-cs ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.3jdzBoXjYz +++ mktemp ++ local LAST_ERR=/tmp/tmp.NgHbe6WMu6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3jdzBoXjYz ++ cat /tmp/tmp.NgHbe6WMu6 ++ rm /tmp/tmp.3jdzBoXjYz /tmp/tmp.NgHbe6WMu6 ++ return 0 + [[ 2 -ne 1 ]] + [[ 2 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount gcp-cs ++ local storage=gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ grep gcp-cs ++ local LAST_OUT=/tmp/tmp.AnFOhr97le +++ mktemp ++ local LAST_ERR=/tmp/tmp.8ibfMXgK1l ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AnFOhr97le ++ cat /tmp/tmp.8ibfMXgK1l ++ rm /tmp/tmp.AnFOhr97le /tmp/tmp.8ibfMXgK1l ++ return 0 + [[ 1 -ne 1 ]] + echo 'Finished to check number 1 of backups for gcp-cs. Timer: 3' Finished to check number 1 of backups for gcp-cs. Timer: 3 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= ++ mktemp + local LAST_OUT=/tmp/tmp.QKLPno0uAX + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_ERR=/tmp/tmp.pgrA9ZEnZC + local exit_status=0 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QKLPno0uAX perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.pgrA9ZEnZC + rm /tmp/tmp.QKLPno0uAX /tmp/tmp.pgrA9ZEnZC + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.I9fcMcQ88y +++ mktemp ++ local LAST_ERR=/tmp/tmp.yasRIbfwgO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.I9fcMcQ88y ++ cat /tmp/tmp.yasRIbfwgO ++ rm /tmp/tmp.I9fcMcQ88y /tmp/tmp.yasRIbfwgO ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.eDP7G8J7Km +++ mktemp ++ local LAST_ERR=/tmp/tmp.PQ2RgoIgc5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eDP7G8J7Km ++ cat /tmp/tmp.PQ2RgoIgc5 ++ rm /tmp/tmp.eDP7G8J7Km /tmp/tmp.PQ2RgoIgc5 ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for azure storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for azure storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + kubectl_bin apply -f - + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.1iCLIHu2em ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + local LAST_ERR=/tmp/tmp.z4HRPzAfkc + local exit_status=0 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ seq 0 2 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1iCLIHu2em perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.z4HRPzAfkc + rm /tmp/tmp.1iCLIHu2em /tmp/tmp.z4HRPzAfkc + return 0 + sleep 50 + wait_num_backups azure-blob 2 + local storage=azure-blob + local num=2 + local timer=0 + echo 'Starting to check number 2 of backups for azure-blob' Starting to check number 2 of backups for azure-blob ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.cSOaOe1nyI +++ mktemp ++ local LAST_ERR=/tmp/tmp.RWBJbkrTzV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cSOaOe1nyI ++ cat /tmp/tmp.RWBJbkrTzV ++ rm /tmp/tmp.cSOaOe1nyI /tmp/tmp.RWBJbkrTzV ++ return 0 + [[ 1 -ne 2 ]] + [[ 0 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ wc -l ++ grep -E :Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.evNwgWR4ae +++ mktemp ++ local LAST_ERR=/tmp/tmp.OsTWqehMrb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.evNwgWR4ae ++ cat /tmp/tmp.OsTWqehMrb ++ rm /tmp/tmp.evNwgWR4ae /tmp/tmp.OsTWqehMrb ++ return 0 + [[ 1 -ne 2 ]] + [[ 1 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob +++ mktemp ++ wc -l ++ grep -E :Succeeded ++ local LAST_OUT=/tmp/tmp.c3c1XgATEu +++ mktemp ++ local LAST_ERR=/tmp/tmp.M1dj2Q3ntf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.c3c1XgATEu ++ cat /tmp/tmp.M1dj2Q3ntf ++ rm /tmp/tmp.c3c1XgATEu /tmp/tmp.M1dj2Q3ntf ++ return 0 + [[ 1 -ne 2 ]] + [[ 2 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ grep azure-blob ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -E :Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.kXvXzQuaEQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.CzPSvkdTAh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kXvXzQuaEQ ++ cat /tmp/tmp.CzPSvkdTAh ++ rm /tmp/tmp.kXvXzQuaEQ /tmp/tmp.CzPSvkdTAh ++ return 0 + [[ 1 -ne 2 ]] + [[ 3 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ grep azure-blob ++ local LAST_OUT=/tmp/tmp.HcbNq7wnfp +++ mktemp ++ local LAST_ERR=/tmp/tmp.ufvZlBwtzH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HcbNq7wnfp ++ cat /tmp/tmp.ufvZlBwtzH ++ rm /tmp/tmp.HcbNq7wnfp /tmp/tmp.ufvZlBwtzH ++ return 0 + [[ 1 -ne 2 ]] + [[ 4 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob +++ mktemp ++ grep -E :Succeeded ++ wc -l ++ local LAST_OUT=/tmp/tmp.Hs2NsAcFe0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.hMQfdbRKIJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Hs2NsAcFe0 ++ cat /tmp/tmp.hMQfdbRKIJ ++ rm /tmp/tmp.Hs2NsAcFe0 /tmp/tmp.hMQfdbRKIJ ++ return 0 + [[ 1 -ne 2 ]] + [[ 5 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.y9NHpCssLe +++ mktemp ++ local LAST_ERR=/tmp/tmp.992vs5UvLU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.y9NHpCssLe ++ cat /tmp/tmp.992vs5UvLU ++ rm /tmp/tmp.y9NHpCssLe /tmp/tmp.992vs5UvLU ++ return 0 + [[ 1 -ne 2 ]] + [[ 6 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.H1g70s5q98 +++ mktemp ++ local LAST_ERR=/tmp/tmp.XMAIjY0nOl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.H1g70s5q98 ++ cat /tmp/tmp.XMAIjY0nOl ++ rm /tmp/tmp.H1g70s5q98 /tmp/tmp.XMAIjY0nOl ++ return 0 + [[ 1 -ne 2 ]] + [[ 7 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.12Kz4aalFv +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZxbBaDrme4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.12Kz4aalFv ++ cat /tmp/tmp.ZxbBaDrme4 ++ rm /tmp/tmp.12Kz4aalFv /tmp/tmp.ZxbBaDrme4 ++ return 0 + [[ 1 -ne 2 ]] + [[ 8 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob +++ mktemp ++ grep -E :Succeeded ++ wc -l ++ local LAST_OUT=/tmp/tmp.quzKc3KC9O +++ mktemp ++ local LAST_ERR=/tmp/tmp.6nH2UCLljF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.quzKc3KC9O ++ cat /tmp/tmp.6nH2UCLljF ++ rm /tmp/tmp.quzKc3KC9O /tmp/tmp.6nH2UCLljF ++ return 0 + [[ 1 -ne 2 ]] + [[ 9 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.nD5Ewt05u1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Cfu5X1YPX0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nD5Ewt05u1 ++ cat /tmp/tmp.Cfu5X1YPX0 ++ rm /tmp/tmp.nD5Ewt05u1 /tmp/tmp.Cfu5X1YPX0 ++ return 0 + [[ 1 -ne 2 ]] + [[ 10 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.pvy4IyGjMv +++ mktemp ++ local LAST_ERR=/tmp/tmp.GVXcKr2x07 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pvy4IyGjMv ++ cat /tmp/tmp.GVXcKr2x07 ++ rm /tmp/tmp.pvy4IyGjMv /tmp/tmp.GVXcKr2x07 ++ return 0 + [[ 1 -ne 2 ]] + [[ 11 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ wc -l ++ grep -E :Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.v35eG2FycL +++ mktemp ++ local LAST_ERR=/tmp/tmp.eMwihEgrhV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.v35eG2FycL ++ cat /tmp/tmp.eMwihEgrhV ++ rm /tmp/tmp.v35eG2FycL /tmp/tmp.eMwihEgrhV ++ return 0 + [[ 1 -ne 2 ]] + [[ 12 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.5gBtjtV1TB +++ mktemp ++ local LAST_ERR=/tmp/tmp.nWuSj9vjvT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5gBtjtV1TB ++ cat /tmp/tmp.nWuSj9vjvT ++ rm /tmp/tmp.5gBtjtV1TB /tmp/tmp.nWuSj9vjvT ++ return 0 + [[ 2 -ne 2 ]] + echo 'Finished to check number 2 of backups for azure-blob. Timer: 13' Finished to check number 2 of backups for azure-blob. Timer: 13 + echo 'Patch backup for yearly backup for azure storag' Patch backup for yearly backup for azure storag + kubectl patch pxc scheduled-backup --type=merge -p '{"spec":{"backup":{"schedule":[{"name":"each-min-azure-every","storageName":"azure-blob","keep":1,"schedule":"0 0 1 * *"}]}}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched + wait_num_backups azure-blob 1 + local storage=azure-blob + local num=1 + local timer=0 + echo 'Starting to check number 1 of backups for azure-blob' Starting to check number 1 of backups for azure-blob ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.EYKwEQ94s0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.r2tnnTBmYs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EYKwEQ94s0 ++ cat /tmp/tmp.r2tnnTBmYs ++ rm /tmp/tmp.EYKwEQ94s0 /tmp/tmp.r2tnnTBmYs ++ return 0 + [[ 2 -ne 1 ]] + [[ 0 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ grep azure-blob ++ wc -l ++ grep -E :Succeeded ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Mp0vbqp7qw +++ mktemp ++ local LAST_ERR=/tmp/tmp.DeZPMpaZxv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Mp0vbqp7qw ++ cat /tmp/tmp.DeZPMpaZxv ++ rm /tmp/tmp.Mp0vbqp7qw /tmp/tmp.DeZPMpaZxv ++ return 0 + [[ 2 -ne 1 ]] + [[ 1 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount azure-blob ++ local storage=azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep azure-blob ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.q2rSN8sdb3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.p8tKNC5qLi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.q2rSN8sdb3 ++ cat /tmp/tmp.p8tKNC5qLi ++ rm /tmp/tmp.q2rSN8sdb3 /tmp/tmp.p8tKNC5qLi ++ return 0 + [[ 1 -ne 1 ]] + echo 'Finished to check number 1 of backups for azure-blob. Timer: 2' Finished to check number 1 of backups for azure-blob. Timer: 2 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + local LAST_OUT=/tmp/tmp.FlLYdqf44j + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + local LAST_ERR=/tmp/tmp.MsUYxz1PPO + local exit_status=0 + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FlLYdqf44j perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.MsUYxz1PPO + rm /tmp/tmp.FlLYdqf44j /tmp/tmp.MsUYxz1PPO + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.ztCmgwxEwE +++ mktemp ++ local LAST_ERR=/tmp/tmp.wltTrp1FUc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ztCmgwxEwE ++ cat /tmp/tmp.wltTrp1FUc ++ rm /tmp/tmp.ztCmgwxEwE /tmp/tmp.wltTrp1FUc ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Nr0PAdfsq5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.MP26a5iKme ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Nr0PAdfsq5 ++ cat /tmp/tmp.MP26a5iKme ++ rm /tmp/tmp.Nr0PAdfsq5 /tmp/tmp.MP26a5iKme ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.LTZPQs5gVn ++ mktemp + local LAST_ERR=/tmp/tmp.RyGt7mOsMI + local exit_status=0 + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + for i in '$(seq 0 2)' + set +e + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + kubectl apply -f - + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LTZPQs5gVn perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.RyGt7mOsMI + rm /tmp/tmp.LTZPQs5gVn /tmp/tmp.RyGt7mOsMI + return 0 + sleep 50 + wait_num_backups aws-s3 2 + local storage=aws-s3 + local num=2 + local timer=0 + echo 'Starting to check number 2 of backups for aws-s3' Starting to check number 2 of backups for aws-s3 ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.DDOPV31BFf +++ mktemp ++ local LAST_ERR=/tmp/tmp.ozS2uJOdbG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DDOPV31BFf ++ cat /tmp/tmp.ozS2uJOdbG ++ rm /tmp/tmp.DDOPV31BFf /tmp/tmp.ozS2uJOdbG ++ return 0 + [[ 1 -ne 2 ]] + [[ 0 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.l0dvplOIoR +++ mktemp ++ local LAST_ERR=/tmp/tmp.4dP0dJw2l7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.l0dvplOIoR ++ cat /tmp/tmp.4dP0dJw2l7 ++ rm /tmp/tmp.l0dvplOIoR /tmp/tmp.4dP0dJw2l7 ++ return 0 + [[ 1 -ne 2 ]] + [[ 1 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep aws-s3 ++ grep -E :Succeeded ++ local LAST_OUT=/tmp/tmp.Gvvn0fnax8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.i0TcshSrFF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Gvvn0fnax8 ++ cat /tmp/tmp.i0TcshSrFF ++ rm /tmp/tmp.Gvvn0fnax8 /tmp/tmp.i0TcshSrFF ++ return 0 + [[ 1 -ne 2 ]] + [[ 2 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep -E :Succeeded +++ mktemp ++ grep aws-s3 ++ local LAST_OUT=/tmp/tmp.A4u2IS1NAI +++ mktemp ++ local LAST_ERR=/tmp/tmp.ecqZiZsttz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A4u2IS1NAI ++ cat /tmp/tmp.ecqZiZsttz ++ rm /tmp/tmp.A4u2IS1NAI /tmp/tmp.ecqZiZsttz ++ return 0 + [[ 1 -ne 2 ]] + [[ 3 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.Pbes09cag5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.36iealnHFN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Pbes09cag5 ++ cat /tmp/tmp.36iealnHFN ++ rm /tmp/tmp.Pbes09cag5 /tmp/tmp.36iealnHFN ++ return 0 + [[ 1 -ne 2 ]] + [[ 4 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ wc -l ++ grep -E :Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.IvHrWBvPDE +++ mktemp ++ local LAST_ERR=/tmp/tmp.i72UelddTt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IvHrWBvPDE ++ cat /tmp/tmp.i72UelddTt ++ rm /tmp/tmp.IvHrWBvPDE /tmp/tmp.i72UelddTt ++ return 0 + [[ 1 -ne 2 ]] + [[ 5 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ grep -E :Succeeded ++ grep aws-s3 ++ local LAST_OUT=/tmp/tmp.wOML8b7qVN ++ wc -l +++ mktemp ++ local LAST_ERR=/tmp/tmp.5n7UFel8pL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wOML8b7qVN ++ cat /tmp/tmp.5n7UFel8pL ++ rm /tmp/tmp.wOML8b7qVN /tmp/tmp.5n7UFel8pL ++ return 0 + [[ 1 -ne 2 ]] + [[ 6 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ grep -E :Succeeded ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.rx2PFAszz5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.kaaGBwbQOU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rx2PFAszz5 ++ cat /tmp/tmp.kaaGBwbQOU ++ rm /tmp/tmp.rx2PFAszz5 /tmp/tmp.kaaGBwbQOU ++ return 0 + [[ 1 -ne 2 ]] + [[ 7 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.5MQWHJ0sXt +++ mktemp ++ local LAST_ERR=/tmp/tmp.OQDYRBOh0N ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5MQWHJ0sXt ++ cat /tmp/tmp.OQDYRBOh0N ++ rm /tmp/tmp.5MQWHJ0sXt /tmp/tmp.OQDYRBOh0N ++ return 0 + [[ 1 -ne 2 ]] + [[ 8 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 +++ mktemp ++ grep -E :Succeeded ++ wc -l ++ local LAST_OUT=/tmp/tmp.1ss8kpjk1d +++ mktemp ++ local LAST_ERR=/tmp/tmp.xt8ecbWhrI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1ss8kpjk1d ++ cat /tmp/tmp.xt8ecbWhrI ++ rm /tmp/tmp.1ss8kpjk1d /tmp/tmp.xt8ecbWhrI ++ return 0 + [[ 1 -ne 2 ]] + [[ 9 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.hYtv6bddcI +++ mktemp ++ local LAST_ERR=/tmp/tmp.Af3byjFePZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hYtv6bddcI ++ cat /tmp/tmp.Af3byjFePZ ++ rm /tmp/tmp.hYtv6bddcI /tmp/tmp.Af3byjFePZ ++ return 0 + [[ 1 -ne 2 ]] + [[ 10 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ wc -l ++ grep -E :Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.s5GxuOAe31 +++ mktemp ++ local LAST_ERR=/tmp/tmp.HcLiH7rYA4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.s5GxuOAe31 ++ cat /tmp/tmp.HcLiH7rYA4 ++ rm /tmp/tmp.s5GxuOAe31 /tmp/tmp.HcLiH7rYA4 ++ return 0 + [[ 1 -ne 2 ]] + [[ 11 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.owx6U7rv9S +++ mktemp ++ local LAST_ERR=/tmp/tmp.Q1UvOfMnOi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.owx6U7rv9S ++ cat /tmp/tmp.Q1UvOfMnOi ++ rm /tmp/tmp.owx6U7rv9S /tmp/tmp.Q1UvOfMnOi ++ return 0 + [[ 1 -ne 2 ]] + [[ 12 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 +++ mktemp ++ wc -l ++ grep -E :Succeeded ++ local LAST_OUT=/tmp/tmp.S8RPXqrsHS +++ mktemp ++ local LAST_ERR=/tmp/tmp.8MWHczq0l0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.S8RPXqrsHS ++ cat /tmp/tmp.8MWHczq0l0 ++ rm /tmp/tmp.S8RPXqrsHS /tmp/tmp.8MWHczq0l0 ++ return 0 + [[ 1 -ne 2 ]] + [[ 13 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.a89KboS4ug +++ mktemp ++ local LAST_ERR=/tmp/tmp.qzomSyNoBu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.a89KboS4ug ++ cat /tmp/tmp.qzomSyNoBu ++ rm /tmp/tmp.a89KboS4ug /tmp/tmp.qzomSyNoBu ++ return 0 + [[ 1 -ne 2 ]] + [[ 14 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.LdJ50vAufg +++ mktemp ++ local LAST_ERR=/tmp/tmp.H9yRrQIzOv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LdJ50vAufg ++ cat /tmp/tmp.H9yRrQIzOv ++ rm /tmp/tmp.LdJ50vAufg /tmp/tmp.H9yRrQIzOv ++ return 0 + [[ 1 -ne 2 ]] + [[ 15 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.63dh1A2cuO +++ mktemp ++ local LAST_ERR=/tmp/tmp.WievFtCN4f ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.63dh1A2cuO ++ cat /tmp/tmp.WievFtCN4f ++ rm /tmp/tmp.63dh1A2cuO /tmp/tmp.WievFtCN4f ++ return 0 + [[ 2 -ne 2 ]] + echo 'Finished to check number 2 of backups for aws-s3. Timer: 16' Finished to check number 2 of backups for aws-s3. Timer: 16 + echo 'Patch backup for yearly backup for aws s3 storage' Patch backup for yearly backup for aws s3 storage + kubectl patch pxc scheduled-backup --type=merge -p '{"spec":{"backup":{"schedule":[{"name":"each-min-aws-s3","storageName":"aws-s3","keep":1,"schedule":"0 0 1 * *"}]}}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched + wait_num_backups aws-s3 1 + local storage=aws-s3 + local num=1 + local timer=0 + echo 'Starting to check number 1 of backups for aws-s3' Starting to check number 1 of backups for aws-s3 ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -E :Succeeded ++ grep aws-s3 +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.Y9wm6JK3St +++ mktemp ++ local LAST_ERR=/tmp/tmp.4YClQVMx7u ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Y9wm6JK3St ++ cat /tmp/tmp.4YClQVMx7u ++ rm /tmp/tmp.Y9wm6JK3St /tmp/tmp.4YClQVMx7u ++ return 0 + [[ 2 -ne 1 ]] + [[ 0 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ grep aws-s3 ++ grep -E :Succeeded ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kITgzQyIbs +++ mktemp ++ local LAST_ERR=/tmp/tmp.yz03NqfAQz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kITgzQyIbs ++ cat /tmp/tmp.yz03NqfAQz ++ rm /tmp/tmp.kITgzQyIbs /tmp/tmp.yz03NqfAQz ++ return 0 + [[ 2 -ne 1 ]] + [[ 1 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.dmN93Wzu0v +++ mktemp ++ local LAST_ERR=/tmp/tmp.1cMESIa3cU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dmN93Wzu0v ++ cat /tmp/tmp.1cMESIa3cU ++ rm /tmp/tmp.dmN93Wzu0v /tmp/tmp.1cMESIa3cU ++ return 0 + [[ 2 -ne 1 ]] + [[ 2 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.vjB8cn6gEV +++ mktemp ++ local LAST_ERR=/tmp/tmp.jmYSnL2MsN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vjB8cn6gEV ++ cat /tmp/tmp.jmYSnL2MsN ++ rm /tmp/tmp.vjB8cn6gEV /tmp/tmp.jmYSnL2MsN ++ return 0 + [[ 2 -ne 1 ]] + [[ 3 -le 400 ]] + sleep 2 + (( timer += 1 )) ++ get_successful_backups_amount aws-s3 ++ local storage=aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep aws-s3 ++ grep -E :Succeeded ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.FlIZ9goid3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.OuHQdlJfZ3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FlIZ9goid3 ++ cat /tmp/tmp.OuHQdlJfZ3 ++ rm /tmp/tmp.FlIZ9goid3 /tmp/tmp.OuHQdlJfZ3 ++ return 0 + [[ 1 -ne 1 ]] + echo 'Finished to check number 1 of backups for aws-s3. Timer: 4' Finished to check number 1 of backups for aws-s3. Timer: 4 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + local LAST_OUT=/tmp/tmp.4NTQzMAm8h ++ mktemp + local LAST_ERR=/tmp/tmp.MN996arLeE + local exit_status=0 + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4NTQzMAm8h perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.MN996arLeE + rm /tmp/tmp.4NTQzMAm8h /tmp/tmp.MN996arLeE + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.1PqhTz0trh +++ mktemp ++ local LAST_ERR=/tmp/tmp.jtKoN8Adld ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1PqhTz0trh ++ cat /tmp/tmp.jtKoN8Adld ++ rm /tmp/tmp.1PqhTz0trh /tmp/tmp.jtKoN8Adld ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.P9lubVwN7g +++ mktemp ++ local LAST_ERR=/tmp/tmp.StkKs5YNwa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.P9lubVwN7g ++ cat /tmp/tmp.StkKs5YNwa ++ rm /tmp/tmp.P9lubVwN7g /tmp/tmp.StkKs5YNwa ++ return 0 + [[ 0 -gt 0 ]] + sleep 30 ++ head -n1 ++ grep aws ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.G7eE0B6ZTZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZKHG9zcJgv ++ local exit_status=0 +++ seq 0 2 ++ cut -d: -f1 ++ grep Succeeded ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.G7eE0B6ZTZ ++ cat /tmp/tmp.ZKHG9zcJgv ++ rm /tmp/tmp.G7eE0B6ZTZ /tmp/tmp.ZKHG9zcJgv ++ return 0 + SECOND_AWS_BACKUP=cron-scheduled-backup-aws-s3-202651622441-q6fav ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ grep gcp ++ head -n1 ++ cut -d: -f1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.3LtIEtt37O +++ mktemp ++ local LAST_ERR=/tmp/tmp.0SKBFO8EyF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3LtIEtt37O ++ cat /tmp/tmp.0SKBFO8EyF ++ rm /tmp/tmp.3LtIEtt37O /tmp/tmp.0SKBFO8EyF ++ return 0 + SECOND_GCP_BACKUP=cron-scheduled-backup-gcp-cs-202651622056-q6fav ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ head -n1 +++ mktemp ++ grep azure ++ cut -d: -f1 ++ local LAST_OUT=/tmp/tmp.k7CnbAsCJA +++ mktemp ++ local LAST_ERR=/tmp/tmp.5OkH4WUeON ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.k7CnbAsCJA ++ cat /tmp/tmp.5OkH4WUeON ++ rm /tmp/tmp.k7CnbAsCJA /tmp/tmp.5OkH4WUeON ++ return 0 + SECOND_AZURE_BACKUP=cron-scheduled-backup-azure-blob-202651622254-q6fav + [[ cron-scheduled-backup-gcp-cs-2026516215632-q6fav == \c\r\o\n\-\s\c\h\e\d\u\l\e\d\-\b\a\c\k\u\p\-\g\c\p\-\c\s\-\2\0\2\6\5\1\6\2\2\0\5\6\-\q\6\f\a\v ]] + [[ cron-scheduled-backup-azure-blob-2026516215818-q6fav == \c\r\o\n\-\s\c\h\e\d\u\l\e\d\-\b\a\c\k\u\p\-\a\z\u\r\e\-\b\l\o\b\-\2\0\2\6\5\1\6\2\2\2\5\4\-\q\6\f\a\v ]] + [[ cron-scheduled-backup-aws-s3-202651621525-q6fav == \c\r\o\n\-\s\c\h\e\d\u\l\e\d\-\b\a\c\k\u\p\-\a\w\s\-\s\3\-\2\0\2\6\5\1\6\2\2\4\4\1\-\q\6\f\a\v ]] + desc 'Check backup deletion' + set +o xtrace ----------------------------------------------------------------------------------- Check backup deletion ----------------------------------------------------------------------------------- + check_backup_deletion_aws operator-testing/scheduled-backup-2026-05-16-21:52:05-full .sst_info/sst_info.00000000000000000000 ++ echo operator-testing/scheduled-backup-2026-05-16-21:52:05-full ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/scheduled-backup-2026-05-16-21:52:05-full ++ cut -d/ -f2- + key_prefix=scheduled-backup-2026-05-16-21:52:05-full + key=.sst_info/sst_info.00000000000000000000 + storage_name=aws-s3 + retry=0 + aws s3api head-object --bucket operator-testing --key scheduled-backup-2026-05-16-21:52:05-full.sst_info/sst_info.00000000000000000000 + echo 'Backup scheduled-backup-2026-05-16-21:52:05-full.sst_info/sst_info.00000000000000000000 in bucket operator-testing not found in aws-s3' Backup scheduled-backup-2026-05-16-21:52:05-full.sst_info/sst_info.00000000000000000000 in bucket operator-testing not found in aws-s3 + check_backup_deletion_gcs operator-testing/scheduled-backup-2026-05-16-21:56:32-full + backup_dest_gcp=operator-testing/scheduled-backup-2026-05-16-21:56:32-full + storage_name=gcp-cs + retry=0 + key=.sst_info/sst_info.00000000000000000000 + gcs_path=gs://operator-testing/scheduled-backup-2026-05-16-21:56:32-full.sst_info/sst_info.00000000000000000000 + gsutil ls gs://operator-testing/scheduled-backup-2026-05-16-21:56:32-full.sst_info/sst_info.00000000000000000000 + echo 'Backup gs://operator-testing/scheduled-backup-2026-05-16-21:56:32-full.sst_info/sst_info.00000000000000000000 not found in gcp-cs' Backup gs://operator-testing/scheduled-backup-2026-05-16-21:56:32-full.sst_info/sst_info.00000000000000000000 not found in gcp-cs + check_backup_deletion_azure operator-testing/scheduled-backup-2026-05-16-21:58:18-full .sst_info/sst_info.00000000000000000000 ++ echo operator-testing/scheduled-backup-2026-05-16-21:58:18-full ++ cut -d/ -f1 + container=operator-testing ++ echo operator-testing/scheduled-backup-2026-05-16-21:58:18-full ++ cut -d/ -f2- + blob_prefix=scheduled-backup-2026-05-16-21:58:18-full + blob=.sst_info/sst_info.00000000000000000000 + storage_name=azure-blob + retry=0 + blob_path=scheduled-backup-2026-05-16-21:58:18-full.sst_info/sst_info.00000000000000000000 + az storage blob show --container-name operator-testing --name scheduled-backup-2026-05-16-21:58:18-full.sst_info/sst_info.00000000000000000000 + echo 'Backup scheduled-backup-2026-05-16-21:58:18-full.sst_info/sst_info.00000000000000000000 in container operator-testing not found in azure-blob' Backup scheduled-backup-2026-05-16-21:58:18-full.sst_info/sst_info.00000000000000000000 in container operator-testing not found in azure-blob ++ get_backup_name pvc ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :pvc:Succeeded ++ cut -d : -f 1 ++ tail -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.2WXZLZMk43 +++ mktemp ++ local LAST_ERR=/tmp/tmp.TEgc7haea2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2WXZLZMk43 ++ cat /tmp/tmp.TEgc7haea2 ++ rm /tmp/tmp.2WXZLZMk43 /tmp/tmp.TEgc7haea2 ++ return 0 + backup_name_pvc=cron-scheduled-backup-pvc-2026516214959-q6fav ++ get_backup_name minio ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ tail -1 ++ grep :minio:Succeeded +++ mktemp ++ cut -d : -f 1 ++ local LAST_OUT=/tmp/tmp.4pRHII41Of +++ mktemp ++ local LAST_ERR=/tmp/tmp.Fn9v4xSFH5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4pRHII41Of ++ cat /tmp/tmp.Fn9v4xSFH5 ++ rm /tmp/tmp.4pRHII41Of /tmp/tmp.Fn9v4xSFH5 ++ return 0 + backup_name_minio=cron-scheduled-backup-minio-2026516215424-q6fav + '[' -z '' ']' ++ get_backup_name aws-s3 ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ cut -d : -f 1 ++ tail -1 ++ grep :aws-s3:Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.g1VE9sOTrB +++ mktemp ++ local LAST_ERR=/tmp/tmp.KfFqXpav5q ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.g1VE9sOTrB ++ cat /tmp/tmp.KfFqXpav5q ++ rm /tmp/tmp.g1VE9sOTrB /tmp/tmp.KfFqXpav5q ++ return 0 + backup_name_aws=cron-scheduled-backup-aws-s3-202651622441-q6fav ++ get_backup_name gcp-cs ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :gcp-cs:Succeeded ++ tail -1 +++ mktemp ++ cut -d : -f 1 ++ local LAST_OUT=/tmp/tmp.9saffOm0Ls +++ mktemp ++ local LAST_ERR=/tmp/tmp.XevDoqaKrl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9saffOm0Ls ++ cat /tmp/tmp.XevDoqaKrl ++ rm /tmp/tmp.9saffOm0Ls /tmp/tmp.XevDoqaKrl ++ return 0 + backup_name_gcp=cron-scheduled-backup-gcp-cs-202651622056-q6fav ++ get_backup_name azure-blob ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :azure-blob:Succeeded +++ mktemp ++ tail -1 ++ cut -d : -f 1 ++ local LAST_OUT=/tmp/tmp.SWKYqC8z0q +++ mktemp ++ local LAST_ERR=/tmp/tmp.s04CnOYeQ2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SWKYqC8z0q ++ cat /tmp/tmp.s04CnOYeQ2 ++ rm /tmp/tmp.SWKYqC8z0q /tmp/tmp.s04CnOYeQ2 ++ return 0 + backup_name_azure=cron-scheduled-backup-azure-blob-202651622254-q6fav + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local pvc_name= + kubectl_bin apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + local LAST_OUT=/tmp/tmp.ptZNgcd9Op + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-28307~ ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2467-3dc7f023#' + local LAST_ERR=/tmp/tmp.tG90sh76Ea + local exit_status=0 + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2467/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ptZNgcd9Op perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.tG90sh76Ea + rm /tmp/tmp.ptZNgcd9Op /tmp/tmp.tG90sh76Ea + return 0 + run_recovery_check scheduled-backup cron-scheduled-backup-pvc-2026516214959-q6fav + local cluster=scheduled-backup + local backup1=cron-scheduled-backup-pvc-2026516214959-q6fav + restore_name=pvc-2026516214959-q6fav + desc 'write data after backup' + set +o xtrace ----------------------------------------------------------------------------------- write data after backup ----------------------------------------------------------------------------------- + run_mysql 'INSERT myApp.myApp (id) VALUES (100501)' '-h scheduled-backup-proxysql -uroot -proot_password' + local 'command=INSERT myApp.myApp (id) VALUES (100501)' + local 'uri=-h scheduled-backup-proxysql -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.I3sDcogBKs +++ mktemp ++ local LAST_ERR=/tmp/tmp.X0BsRdjMWe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.I3sDcogBKs ++ cat /tmp/tmp.X0BsRdjMWe ++ rm /tmp/tmp.I3sDcogBKs /tmp/tmp.X0BsRdjMWe ++ return 0 + client_pod=pxc-client-67fc4995bb-qtkdl + wait_pod pxc-client-67fc4995bb-qtkdl + local pod=pxc-client-67fc4995bb-qtkdl + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-qtkdl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-qtkdl condition met E0516 22:06:30.647731 12591 reflector.go:227] "Failed to watch" err="Get \"https://34.122.65.69/api/v1/namespaces/scheduled-backup-28307/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-qtkdl&resourceVersion=1778969189904574000&timeoutSeconds=310&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-qtkdl to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 20