Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/logs/scheduled-backup-5-7.log Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 + main + create_infra scheduled-backup-26803 + local ns=scheduled-backup-26803 + '[' -n pxc-operator ']' + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl get pxc --all-namespaces -o wide + kubectl patch pxc -n scheduled-backup-27391 scheduled-backup --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.V6sZNgSPFJ ++ mktemp + local LAST_ERR=/tmp/tmp.HmOhB1vH9t + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.V6sZNgSPFJ perconaxtradbcluster.pxc.percona.com "scheduled-backup" deleted from scheduled-backup-27391 namespace + cat /tmp/tmp.HmOhB1vH9t + rm /tmp/tmp.V6sZNgSPFJ /tmp/tmp.HmOhB1vH9t + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.KUbLkQohPw ++ mktemp + local LAST_ERR=/tmp/tmp.4gltxPtkLv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KUbLkQohPw perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-aws-s3-2025111210101-q6fav" deleted from scheduled-backup-27391 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-azure-blob-20251112101527-q6fav" deleted from scheduled-backup-27391 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-gcp-cs-20251112101338-q6fav" deleted from scheduled-backup-27391 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-minio-20251112101152-q6fav" deleted from scheduled-backup-27391 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-pvc-2025111210737-q6fav" deleted from scheduled-backup-27391 namespace + cat /tmp/tmp.4gltxPtkLv + rm /tmp/tmp.KUbLkQohPw /tmp/tmp.4gltxPtkLv + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.vhZIjdOA9T ++ mktemp + local LAST_ERR=/tmp/tmp.2ATo3OtQ2e + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vhZIjdOA9T No resources found + cat /tmp/tmp.2ATo3OtQ2e + rm /tmp/tmp.vhZIjdOA9T /tmp/tmp.2ATo3OtQ2e + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + xargs kubectl delete ns + awk '{print$1}' + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.hKYiJGnNj8 + local LAST_OUT=/tmp/tmp.awxuKgMhnB ++ mktemp + local LAST_ERR=/tmp/tmp.eDvWbRxU4V + local exit_status=0 ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.wr6rfo20SF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hKYiJGnNj8 + cat /tmp/tmp.wr6rfo20SF + rm /tmp/tmp.hKYiJGnNj8 /tmp/tmp.wr6rfo20SF + return 0 namespace "scheduled-backup-27391" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.awxuKgMhnB namespace "pxc-operator" deleted + cat /tmp/tmp.eDvWbRxU4V + rm /tmp/tmp.awxuKgMhnB /tmp/tmp.eDvWbRxU4V + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.S9XFzcltoe ++ mktemp + local LAST_ERR=/tmp/tmp.hXzjTKY7gy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.S9XFzcltoe namespace/pxc-operator created + cat /tmp/tmp.hXzjTKY7gy + rm /tmp/tmp.S9XFzcltoe /tmp/tmp.hXzjTKY7gy + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.2ofc5WM24b +++ mktemp ++ local LAST_ERR=/tmp/tmp.UuoqUVqSD0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2ofc5WM24b ++ cat /tmp/tmp.UuoqUVqSD0 ++ rm /tmp/tmp.2ofc5WM24b /tmp/tmp.UuoqUVqSD0 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2232-b6dbce1b-6-cluster3 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.dbZBKAq77X ++ mktemp + local LAST_ERR=/tmp/tmp.4v09iN2afC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2232-b6dbce1b-6-cluster3 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dbZBKAq77X Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2232-b6dbce1b-6-cluster3" modified. + cat /tmp/tmp.4v09iN2afC + rm /tmp/tmp.dbZBKAq77X /tmp/tmp.4v09iN2afC + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.45QP35bRiS ++ mktemp + local LAST_ERR=/tmp/tmp.vFXIcCL8ln + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.45QP35bRiS customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.vFXIcCL8ln + rm /tmp/tmp.45QP35bRiS /tmp/tmp.vFXIcCL8ln + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: pxc-operator^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/deploy/cw-rbac.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.kHoSJ3s7z7 ++ mktemp + local LAST_ERR=/tmp/tmp.TaIj9UFp0r + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kHoSJ3s7z7 clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.TaIj9UFp0r + rm /tmp/tmp.kHoSJ3s7z7 /tmp/tmp.TaIj9UFp0r + return 0 + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2232-b6dbce1b^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/deploy/cw-operator.yaml + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.wgJNIlmfLz ++ mktemp + local LAST_ERR=/tmp/tmp.SsEQYpc4LZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wgJNIlmfLz deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.SsEQYpc4LZ + rm /tmp/tmp.wgJNIlmfLz /tmp/tmp.SsEQYpc4LZ + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.JNnQ7HnJCs ++ mktemp + local LAST_ERR=/tmp/tmp.WUqBR7TaZA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JNnQ7HnJCs pod/percona-xtradb-cluster-operator-74796f4949-4z8dl condition met + cat /tmp/tmp.WUqBR7TaZA + rm /tmp/tmp.JNnQ7HnJCs /tmp/tmp.WUqBR7TaZA + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.pNuUP5sXCm +++ mktemp ++ local LAST_ERR=/tmp/tmp.yZKYBh4sOp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pNuUP5sXCm ++ cat /tmp/tmp.yZKYBh4sOp ++ rm /tmp/tmp.pNuUP5sXCm /tmp/tmp.yZKYBh4sOp ++ return 0 + wait_pod percona-xtradb-cluster-operator-74796f4949-4z8dl 480 pxc-operator + local pod=percona-xtradb-cluster-operator-74796f4949-4z8dl + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-74796f4949-4z8dl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-74796f4949-4z8dl condition met waiting for pod/percona-xtradb-cluster-operator-74796f4949-4z8dl to become Ready.Ok + sleep 3 + create_namespace scheduled-backup-26803 + local namespace=scheduled-backup-26803 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces scheduled-backup-26803' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces scheduled-backup-26803 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace scheduled-backup-26803 ++ mktemp + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.mzHtpsNQ67 ++ mktemp + local LAST_ERR=/tmp/tmp.0idR9ecf1m + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-26803 + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.atrEIBPorI ++ mktemp + local LAST_ERR=/tmp/tmp.inpTpvqO1h + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-26803 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.atrEIBPorI + cat /tmp/tmp.inpTpvqO1h + rm /tmp/tmp.atrEIBPorI /tmp/tmp.inpTpvqO1h + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-26803 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.mzHtpsNQ67 + cat /tmp/tmp.0idR9ecf1m Error from server (NotFound): namespaces "scheduled-backup-26803" not found + rm /tmp/tmp.mzHtpsNQ67 /tmp/tmp.0idR9ecf1m + return 1 + : + wait_for_delete namespace/scheduled-backup-26803 + local res=namespace/scheduled-backup-26803 + echo -n 'waiting for namespace/scheduled-backup-26803 to be deleted' waiting for namespace/scheduled-backup-26803 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "scheduled-backup-26803" not found + desc 'create namespace scheduled-backup-26803' + set +o xtrace ----------------------------------------------------------------------------------- create namespace scheduled-backup-26803 ----------------------------------------------------------------------------------- + kubectl_bin create namespace scheduled-backup-26803 ++ mktemp + local LAST_OUT=/tmp/tmp.dcjVMK3qla ++ mktemp + local LAST_ERR=/tmp/tmp.0gNImjBJ0b + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace scheduled-backup-26803 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dcjVMK3qla namespace/scheduled-backup-26803 created + cat /tmp/tmp.0gNImjBJ0b + rm /tmp/tmp.dcjVMK3qla /tmp/tmp.0gNImjBJ0b + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.0EJW0k0kHT +++ mktemp ++ local LAST_ERR=/tmp/tmp.1iBU9cDNxc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0EJW0k0kHT ++ cat /tmp/tmp.1iBU9cDNxc ++ rm /tmp/tmp.0EJW0k0kHT /tmp/tmp.1iBU9cDNxc ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2232-b6dbce1b-6-cluster3 --namespace=scheduled-backup-26803 ++ mktemp + local LAST_OUT=/tmp/tmp.WxAmiuvbUd ++ mktemp + local LAST_ERR=/tmp/tmp.KdzIiVp2jf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2232-b6dbce1b-6-cluster3 --namespace=scheduled-backup-26803 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WxAmiuvbUd Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2232-b6dbce1b-6-cluster3" modified. + cat /tmp/tmp.KdzIiVp2jf + rm /tmp/tmp.WxAmiuvbUd /tmp/tmp.KdzIiVp2jf + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.UuhRGQcJhS ++ mktemp + local LAST_ERR=/tmp/tmp.3s3U0l8do5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UuhRGQcJhS secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.3s3U0l8do5 + rm /tmp/tmp.UuhRGQcJhS /tmp/tmp.3s3U0l8do5 + return 0 + start_minio + deploy_helm scheduled-backup-26803 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "hashicorp" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Wed Nov 12 10:24:10 2025 NAMESPACE: scheduled-backup-26803 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.scheduled-backup-26803.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-26803 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-26803 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-26803 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-26803 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KHYR7w0qlF +++ mktemp ++ local LAST_ERR=/tmp/tmp.J2OSCbqMdr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KHYR7w0qlF ++ cat /tmp/tmp.J2OSCbqMdr ++ rm /tmp/tmp.KHYR7w0qlF /tmp/tmp.J2OSCbqMdr ++ return 0 + MINIO_POD=minio-service-55fcc5d75f-h468p + wait_pod minio-service-55fcc5d75f-h468p + local pod=minio-service-55fcc5d75f-h468p + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo minio-service-55fcc5d75f-h468p + local container= + set +o xtrace pod/minio-service-55fcc5d75f-h468p condition met waiting for pod/minio-service-55fcc5d75f-h468p to become Ready.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.a5AuGnFmHy ++ mktemp + local LAST_ERR=/tmp/tmp.Y98TJTLmhX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.a5AuGnFmHy make_bucket: operator-testing pod "aws-cli" deleted from scheduled-backup-26803 namespace + cat /tmp/tmp.Y98TJTLmhX All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_scheduled-backup-26803 + rm /tmp/tmp.a5AuGnFmHy /tmp/tmp.Y98TJTLmhX + return 0 + cluster=scheduled-backup + cat - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.tdUrcu5VHH ++ mktemp + local LAST_ERR=/tmp/tmp.cgPuwp8drO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tdUrcu5VHH priorityclass.scheduling.k8s.io/high-priority configured + cat /tmp/tmp.cgPuwp8drO + rm /tmp/tmp.tdUrcu5VHH /tmp/tmp.cgPuwp8drO + return 0 + spinup_pxc scheduled-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local cluster=scheduled-backup + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.CH0OAEzAoH ++ mktemp + local LAST_ERR=/tmp/tmp.Nsz1yVUkc7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CH0OAEzAoH secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.Nsz1yVUkc7 + rm /tmp/tmp.CH0OAEzAoH /tmp/tmp.Nsz1yVUkc7 + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/conf/client.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.qxJlH3JFWr + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2232-b6dbce1b#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-26803~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + local LAST_ERR=/tmp/tmp.A6FbKD168O + local exit_status=0 + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qxJlH3JFWr deployment.apps/pxc-client created + cat /tmp/tmp.A6FbKD168O + rm /tmp/tmp.qxJlH3JFWr /tmp/tmp.A6FbKD168O + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local pvc_name= ++ mktemp + local LAST_OUT=/tmp/tmp.z2c9xjAZKY + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_ERR=/tmp/tmp.FCzZw364or + local exit_status=0 + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ seq 0 2 + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-26803~ + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2232-b6dbce1b#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.z2c9xjAZKY perconaxtradbcluster.pxc.percona.com/scheduled-backup created + cat /tmp/tmp.FCzZw364or + rm /tmp/tmp.z2c9xjAZKY /tmp/tmp.FCzZw364or + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy scheduled-backup ++ local target_cluster=scheduled-backup +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sN5gyWq3V3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3htqq0kXO4 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.sN5gyWq3V3 +++ cat /tmp/tmp.3htqq0kXO4 +++ rm /tmp/tmp.sN5gyWq3V3 /tmp/tmp.3htqq0kXO4 +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QWe8g6dJjv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fywq9PNRNB +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.QWe8g6dJjv +++ cat /tmp/tmp.fywq9PNRNB +++ rm /tmp/tmp.QWe8g6dJjv /tmp/tmp.fywq9PNRNB +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scheduled-backup-proxysql ++ return + local proxy=scheduled-backup-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-26803 ++ mktemp + local LAST_OUT=/tmp/tmp.Per5LPysAI ++ mktemp + local LAST_ERR=/tmp/tmp.CugB6xB9K3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-26803 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-26803 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-26803 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.Per5LPysAI + cat /tmp/tmp.CugB6xB9K3 error: no matching resources found + rm /tmp/tmp.Per5LPysAI /tmp/tmp.CugB6xB9K3 + return 1 + true + wait_for_running scheduled-backup-proxysql 1 + local name=scheduled-backup-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-proxysql-0 480 + local pod=scheduled-backup-proxysql-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/scheduled-backup-proxysql-0 condition met waiting for pod/scheduled-backup-proxysql-0 to become Ready.Ok + wait_for_running scheduled-backup-pxc 3 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 480 + local pod=scheduled-backup-pxc-0 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-0 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 480 + local pod=scheduled-backup-pxc-1 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-1 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 480 + local pod=scheduled-backup-pxc-2 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.magjnEAkhA +++ mktemp ++ local LAST_ERR=/tmp/tmp.oywspOG0wJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.magjnEAkhA ++ cat /tmp/tmp.oywspOG0wJ ++ rm /tmp/tmp.magjnEAkhA /tmp/tmp.oywspOG0wJ ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lAheT5PHF6 +++ mktemp ++ egrep -o 'early-plugin-load=keyring_\w+.so' ++ local LAST_ERR=/tmp/tmp.jlGWsjYikK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lAheT5PHF6 ++ cat /tmp/tmp.jlGWsjYikK Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.lAheT5PHF6 /tmp/tmp.jlGWsjYikK ++ return 0 + [[ -n '' ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Em49YntxYO +++ mktemp ++ local LAST_ERR=/tmp/tmp.FM0xkQEba8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Em49YntxYO ++ cat /tmp/tmp.FM0xkQEba8 ++ rm /tmp/tmp.Em49YntxYO /tmp/tmp.FM0xkQEba8 ++ return 0 + client_pod=pxc-client-857d976497-b9vjh + wait_pod pxc-client-857d976497-b9vjh + local pod=pxc-client-857d976497-b9vjh + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-b9vjh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-857d976497-b9vjh condition met waiting for pod/pxc-client-857d976497-b9vjh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mN1VKNbsVE +++ mktemp ++ local LAST_ERR=/tmp/tmp.PQjqN6yNop ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mN1VKNbsVE ++ cat /tmp/tmp.PQjqN6yNop ++ rm /tmp/tmp.mN1VKNbsVE /tmp/tmp.PQjqN6yNop ++ return 0 + client_pod=pxc-client-857d976497-b9vjh + wait_pod pxc-client-857d976497-b9vjh + local pod=pxc-client-857d976497-b9vjh + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-b9vjh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-857d976497-b9vjh condition met waiting for pod/pxc-client-857d976497-b9vjh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GurZqvIW8n +++ mktemp ++ local LAST_ERR=/tmp/tmp.GjukUMFNC3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GurZqvIW8n ++ cat /tmp/tmp.GjukUMFNC3 ++ rm /tmp/tmp.GurZqvIW8n /tmp/tmp.GjukUMFNC3 ++ return 0 + client_pod=pxc-client-857d976497-b9vjh + wait_pod pxc-client-857d976497-b9vjh + local pod=pxc-client-857d976497-b9vjh + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-b9vjh ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-857d976497-b9vjh condition met waiting for pod/pxc-client-857d976497-b9vjh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.QlKBR9yV9D/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.QlKBR9yV9D/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RGJ9GDKSms +++ mktemp ++ local LAST_ERR=/tmp/tmp.5mFrgxvbl1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RGJ9GDKSms ++ cat /tmp/tmp.5mFrgxvbl1 ++ rm /tmp/tmp.RGJ9GDKSms /tmp/tmp.5mFrgxvbl1 ++ return 0 + client_pod=pxc-client-857d976497-b9vjh + wait_pod pxc-client-857d976497-b9vjh + local pod=pxc-client-857d976497-b9vjh + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-b9vjh ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-857d976497-b9vjh condition met waiting for pod/pxc-client-857d976497-b9vjh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.QlKBR9yV9D/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.QlKBR9yV9D/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eCgMC4Dw85 +++ mktemp ++ local LAST_ERR=/tmp/tmp.HCnPEvEmMG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eCgMC4Dw85 ++ cat /tmp/tmp.HCnPEvEmMG ++ rm /tmp/tmp.eCgMC4Dw85 /tmp/tmp.HCnPEvEmMG ++ return 0 + client_pod=pxc-client-857d976497-b9vjh + wait_pod pxc-client-857d976497-b9vjh + local pod=pxc-client-857d976497-b9vjh + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-b9vjh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-857d976497-b9vjh condition met waiting for pod/pxc-client-857d976497-b9vjh to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.QlKBR9yV9D/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.QlKBR9yV9D/select-1.sql ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DWMofMaIF9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.crxo7HbUFR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DWMofMaIF9 ++ cat /tmp/tmp.crxo7HbUFR Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.DWMofMaIF9 /tmp/tmp.crxo7HbUFR ++ return 0 + '[' '' ']' + sleep 20 + desc 'add backups schedule for pvc storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for pvc storage ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.TsGmAvEmAN +++ mktemp ++ local LAST_ERR=/tmp/tmp.u47dZts2XR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TsGmAvEmAN ++ cat /tmp/tmp.u47dZts2XR ++ rm /tmp/tmp.TsGmAvEmAN /tmp/tmp.u47dZts2XR ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2232-b6dbce1b-6-cluster3 --namespace=scheduled-backup-26803 ++ mktemp + local LAST_OUT=/tmp/tmp.ZbOp7aaOBl ++ mktemp + local LAST_ERR=/tmp/tmp.daqMoRBifu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2232-b6dbce1b-6-cluster3 --namespace=scheduled-backup-26803 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZbOp7aaOBl Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2232-b6dbce1b-6-cluster3" modified. + cat /tmp/tmp.daqMoRBifu + rm /tmp/tmp.ZbOp7aaOBl /tmp/tmp.daqMoRBifu + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml '' + kubectl_bin apply -f - + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2232-b6dbce1b#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-26803~ ++ mktemp + local LAST_OUT=/tmp/tmp.bTFTtOGCL9 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + local LAST_ERR=/tmp/tmp.xFvwMkZYxI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bTFTtOGCL9 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.xFvwMkZYxI + rm /tmp/tmp.bTFTtOGCL9 /tmp/tmp.xFvwMkZYxI + return 0 + label_node ++ kubectl_bin get nodes --no-headers=true ++ head -n1 ++ grep -v master +++ mktemp ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.XQSaqQBq3A +++ mktemp ++ local LAST_ERR=/tmp/tmp.iYfyZ1RFIO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get nodes --no-headers=true ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XQSaqQBq3A ++ cat /tmp/tmp.iYfyZ1RFIO ++ rm /tmp/tmp.XQSaqQBq3A /tmp/tmp.iYfyZ1RFIO ++ return 0 + LABELED_NODE=gke-jen-pxc-2232-b6dbce1-default-pool-11385507-crwx + kubectl_bin label nodes gke-jen-pxc-2232-b6dbce1-default-pool-11385507-crwx backupWorker=True --overwrite ++ mktemp + local LAST_OUT=/tmp/tmp.PYsGqIvXjH ++ mktemp + local LAST_ERR=/tmp/tmp.RMVuqSWf6P + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label nodes gke-jen-pxc-2232-b6dbce1-default-pool-11385507-crwx backupWorker=True --overwrite + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PYsGqIvXjH node/gke-jen-pxc-2232-b6dbce1-default-pool-11385507-crwx not labeled + cat /tmp/tmp.RMVuqSWf6P + rm /tmp/tmp.PYsGqIvXjH /tmp/tmp.RMVuqSWf6P + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2232-b6dbce1b#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-26803~ ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.bWsTzla9JK ++ mktemp + local LAST_ERR=/tmp/tmp.V3M31MJBQH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bWsTzla9JK perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.V3M31MJBQH + rm /tmp/tmp.bWsTzla9JK /tmp/tmp.V3M31MJBQH + return 0 + wait_all_backups ++ get_running_backups_amount ++ grep -vE ':Succeeded|:Failed' ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Gn9lssDyz1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.n9TPXXybj2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Gn9lssDyz1 ++ cat /tmp/tmp.n9TPXXybj2 ++ rm /tmp/tmp.Gn9lssDyz1 /tmp/tmp.n9TPXXybj2 ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep :Failed ++ local LAST_OUT=/tmp/tmp.YDYCHYnDpZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.mAUQsKCyQn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YDYCHYnDpZ ++ cat /tmp/tmp.mAUQsKCyQn ++ rm /tmp/tmp.YDYCHYnDpZ /tmp/tmp.mAUQsKCyQn ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.TgyHwjigYZ ++ mktemp + local LAST_ERR=/tmp/tmp.A12tgGgDrT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TgyHwjigYZ NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-20251112103149-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112103149-q6fav-20251112103149-ac645420 Running 26s + cat /tmp/tmp.A12tgGgDrT + rm /tmp/tmp.TgyHwjigYZ /tmp/tmp.A12tgGgDrT + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.uiDu4RCs5c ++ mktemp + local LAST_ERR=/tmp/tmp.LngJpRj10O + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uiDu4RCs5c NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-h468p 1/1 Running 0 8m2s pxc-client-857d976497-b9vjh 2/2 Running 0 6m47s scheduled-backup-proxysql-0 3/3 Running 0 6m44s scheduled-backup-proxysql-1 3/3 Running 0 6m27s scheduled-backup-pxc-0 1/1 Running 0 6m44s scheduled-backup-pxc-1 1/1 Running 0 5m29s scheduled-backup-pxc-2 1/1 Running 0 4m16s xb-cron-scheduled-backup-pvc-20251112103149-q6fav-crjq5 1/1 Running 0 26s + cat /tmp/tmp.LngJpRj10O + rm /tmp/tmp.uiDu4RCs5c /tmp/tmp.LngJpRj10O + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.0IduN4bNNH +++ mktemp ++ local LAST_ERR=/tmp/tmp.nSwrobhaHU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0IduN4bNNH ++ cat /tmp/tmp.nSwrobhaHU ++ rm /tmp/tmp.0IduN4bNNH /tmp/tmp.nSwrobhaHU ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.HdjRJLVDkr +++ mktemp ++ local LAST_ERR=/tmp/tmp.CojYrvJQ34 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HdjRJLVDkr ++ cat /tmp/tmp.CojYrvJQ34 ++ rm /tmp/tmp.HdjRJLVDkr /tmp/tmp.CojYrvJQ34 ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + kubectl_bin apply -f - + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.KPwIBlA2Mz + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2232-b6dbce1b#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-26803~ + local LAST_ERR=/tmp/tmp.GfEgJgByzH + local exit_status=0 + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KPwIBlA2Mz perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.GfEgJgByzH + rm /tmp/tmp.KPwIBlA2Mz /tmp/tmp.GfEgJgByzH + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.2FcYNiz5xx + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2232-b6dbce1b#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_ERR=/tmp/tmp.J9JW3SUe9U + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local exit_status=0 + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-26803~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2FcYNiz5xx perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.J9JW3SUe9U + rm /tmp/tmp.2FcYNiz5xx /tmp/tmp.J9JW3SUe9U + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QTp2iYawvJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.tfqJ3PeLsn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QTp2iYawvJ ++ cat /tmp/tmp.tfqJ3PeLsn ++ rm /tmp/tmp.QTp2iYawvJ /tmp/tmp.tfqJ3PeLsn ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.NPgHQ8vYvN +++ mktemp ++ local LAST_ERR=/tmp/tmp.bxm8Dfd5DQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NPgHQ8vYvN ++ cat /tmp/tmp.bxm8Dfd5DQ ++ rm /tmp/tmp.NPgHQ8vYvN /tmp/tmp.bxm8Dfd5DQ ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.1lLOoL65ZJ ++ mktemp + local LAST_ERR=/tmp/tmp.2GS35JIchl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1lLOoL65ZJ NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112103340-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-11-12-10:33:40-full Running 17s cron-scheduled-backup-pvc-20251112103149-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112103149-q6fav-20251112103149-ac645420 Succeeded 94s 2m8s + cat /tmp/tmp.2GS35JIchl + rm /tmp/tmp.1lLOoL65ZJ /tmp/tmp.2GS35JIchl + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.Hrj1L0bMly ++ mktemp + local LAST_ERR=/tmp/tmp.nyTYEZAyZY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Hrj1L0bMly NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-h468p 1/1 Running 0 9m46s pxc-client-857d976497-b9vjh 2/2 Running 0 8m31s scheduled-backup-proxysql-0 3/3 Running 0 8m28s scheduled-backup-proxysql-1 3/3 Running 0 8m11s scheduled-backup-pxc-0 1/1 Running 0 8m28s scheduled-backup-pxc-1 1/1 Running 0 7m13s scheduled-backup-pxc-2 1/1 Running 0 6m xb-cron-scheduled-backup-aws-s3-20251112103340-q6fav-kgb6h 1/1 Running 0 19s xb-cron-scheduled-backup-pvc-20251112103149-q6fav-crjq5 0/1 Completed 0 2m10s + cat /tmp/tmp.nyTYEZAyZY + rm /tmp/tmp.Hrj1L0bMly /tmp/tmp.nyTYEZAyZY + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.pnLLWYBUQE +++ mktemp ++ local LAST_ERR=/tmp/tmp.mIevvdwrpk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pnLLWYBUQE ++ cat /tmp/tmp.mIevvdwrpk ++ rm /tmp/tmp.pnLLWYBUQE /tmp/tmp.mIevvdwrpk ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.lrZ2n20JRY +++ mktemp ++ local LAST_ERR=/tmp/tmp.JjsyXwGHVk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lrZ2n20JRY ++ cat /tmp/tmp.JjsyXwGHVk ++ rm /tmp/tmp.lrZ2n20JRY /tmp/tmp.JjsyXwGHVk ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for minio storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for minio storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2232-b6dbce1b#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-26803~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_OUT=/tmp/tmp.tGqMEyDkPA ++ mktemp + local LAST_ERR=/tmp/tmp.rEQJgl1m8S + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tGqMEyDkPA perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.rEQJgl1m8S + rm /tmp/tmp.tGqMEyDkPA /tmp/tmp.rEQJgl1m8S + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + kubectl_bin apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_OUT=/tmp/tmp.cZLz58gieC + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2232-b6dbce1b#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-26803~ ++ mktemp + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.h4W3UfyGy5 + local exit_status=0 + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cZLz58gieC perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.h4W3UfyGy5 + rm /tmp/tmp.cZLz58gieC /tmp/tmp.h4W3UfyGy5 + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.TduQCBpSc2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.hH6JBK7Lx4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TduQCBpSc2 ++ cat /tmp/tmp.hH6JBK7Lx4 ++ rm /tmp/tmp.TduQCBpSc2 /tmp/tmp.hH6JBK7Lx4 ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.8LPfHYkDJS +++ mktemp ++ local LAST_ERR=/tmp/tmp.OzsFwnSf0O ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8LPfHYkDJS ++ cat /tmp/tmp.OzsFwnSf0O ++ rm /tmp/tmp.8LPfHYkDJS /tmp/tmp.OzsFwnSf0O ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.Hbo1IkIOA7 ++ mktemp + local LAST_ERR=/tmp/tmp.90dF4zo93W + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Hbo1IkIOA7 NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112103340-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-11-12-10:33:40-full Succeeded 83s 2m2s cron-scheduled-backup-minio-20251112103525-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-11-12-10:35:25-full Running 17s cron-scheduled-backup-pvc-20251112103149-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112103149-q6fav-20251112103149-ac645420 Succeeded 3m19s 3m53s + cat /tmp/tmp.90dF4zo93W + rm /tmp/tmp.Hbo1IkIOA7 /tmp/tmp.90dF4zo93W + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.VBRVOBAcpa ++ mktemp + local LAST_ERR=/tmp/tmp.oC7inle8b1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VBRVOBAcpa NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-h468p 1/1 Running 0 11m pxc-client-857d976497-b9vjh 2/2 Running 0 10m scheduled-backup-proxysql-0 3/3 Running 0 10m scheduled-backup-proxysql-1 3/3 Running 0 9m56s scheduled-backup-pxc-0 1/1 Running 0 10m scheduled-backup-pxc-1 1/1 Running 0 8m58s scheduled-backup-pxc-2 1/1 Running 0 7m45s xb-cron-scheduled-backup-aws-s3-20251112103340-q6fav-kgb6h 0/1 Completed 0 2m4s xb-cron-scheduled-backup-minio-20251112103525-q6fav-5szdj 1/1 Running 0 19s xb-cron-scheduled-backup-pvc-20251112103149-q6fav-crjq5 0/1 Completed 0 3m55s + cat /tmp/tmp.oC7inle8b1 + rm /tmp/tmp.VBRVOBAcpa /tmp/tmp.oC7inle8b1 + return 0 + sleep 20 ++ get_running_backups_amount ++ wc -l ++ grep -vE ':Succeeded|:Failed' ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5sF3GvQFaw +++ mktemp ++ local LAST_ERR=/tmp/tmp.jumpdLzOn4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5sF3GvQFaw ++ cat /tmp/tmp.jumpdLzOn4 ++ rm /tmp/tmp.5sF3GvQFaw /tmp/tmp.jumpdLzOn4 ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z7QzNEqKqD ++ wc -l +++ mktemp ++ grep :Failed ++ local LAST_ERR=/tmp/tmp.3bIppLHhAM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.z7QzNEqKqD ++ cat /tmp/tmp.3bIppLHhAM ++ rm /tmp/tmp.z7QzNEqKqD /tmp/tmp.3bIppLHhAM ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for gcs storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for gcs storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml '' ++ mktemp + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + local LAST_OUT=/tmp/tmp.Fd5qpD9zxl + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2232-b6dbce1b#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-26803~ + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml ++ mktemp + local LAST_ERR=/tmp/tmp.zFkccNJZXT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Fd5qpD9zxl perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.zFkccNJZXT + rm /tmp/tmp.Fd5qpD9zxl /tmp/tmp.zFkccNJZXT + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= ++ mktemp + local LAST_OUT=/tmp/tmp.eVhpVq4yID + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2232-b6dbce1b#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-26803~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_ERR=/tmp/tmp.miD6GwQBqT + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eVhpVq4yID perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.miD6GwQBqT + rm /tmp/tmp.eVhpVq4yID /tmp/tmp.miD6GwQBqT + return 0 + wait_all_backups ++ get_running_backups_amount ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ local LAST_OUT=/tmp/tmp.o9Sd9WuDom +++ mktemp ++ local LAST_ERR=/tmp/tmp.WXMmyqDEwZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.o9Sd9WuDom ++ cat /tmp/tmp.WXMmyqDEwZ ++ rm /tmp/tmp.o9Sd9WuDom /tmp/tmp.WXMmyqDEwZ ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ grep :Failed ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IAVf8wwmDA +++ mktemp ++ local LAST_ERR=/tmp/tmp.XcOnMynbM0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IAVf8wwmDA ++ cat /tmp/tmp.XcOnMynbM0 ++ rm /tmp/tmp.IAVf8wwmDA /tmp/tmp.XcOnMynbM0 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.RW2P6tz1rV ++ mktemp + local LAST_ERR=/tmp/tmp.sMdWqWVmW2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RW2P6tz1rV NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112103340-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-11-12-10:33:40-full Succeeded 3m26s 4m5s cron-scheduled-backup-gcp-cs-2025111210379-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2025-11-12-10:37:09-full Succeeded 3s 36s cron-scheduled-backup-minio-20251112103525-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-11-12-10:35:25-full Succeeded 112s 2m20s cron-scheduled-backup-pvc-20251112103149-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112103149-q6fav-20251112103149-ac645420 Succeeded 5m22s 5m56s + cat /tmp/tmp.sMdWqWVmW2 + rm /tmp/tmp.RW2P6tz1rV /tmp/tmp.sMdWqWVmW2 + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.tKsRQDS4Tw ++ mktemp + local LAST_ERR=/tmp/tmp.XW4khyavho + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tKsRQDS4Tw NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-h468p 1/1 Running 0 13m pxc-client-857d976497-b9vjh 2/2 Running 0 12m scheduled-backup-proxysql-0 3/3 Running 0 12m scheduled-backup-proxysql-1 3/3 Running 0 11m scheduled-backup-pxc-0 1/1 Running 0 12m scheduled-backup-pxc-1 1/1 Running 0 11m scheduled-backup-pxc-2 1/1 Running 0 9m47s xb-cron-scheduled-backup-aws-s3-20251112103340-q6fav-kgb6h 0/1 Completed 0 4m6s xb-cron-scheduled-backup-gcp-cs-2025111210379-q6fav-jxqv2 0/1 Completed 0 37s xb-cron-scheduled-backup-minio-20251112103525-q6fav-5szdj 0/1 Completed 0 2m21s xb-cron-scheduled-backup-pvc-20251112103149-q6fav-crjq5 0/1 Completed 0 5m57s + cat /tmp/tmp.XW4khyavho + rm /tmp/tmp.tKsRQDS4Tw /tmp/tmp.XW4khyavho + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.XyAj8lykfX +++ mktemp ++ local LAST_ERR=/tmp/tmp.Sh75YrB3TY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XyAj8lykfX ++ cat /tmp/tmp.Sh75YrB3TY ++ rm /tmp/tmp.XyAj8lykfX /tmp/tmp.Sh75YrB3TY ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.oKT3hDB9hr +++ mktemp ++ local LAST_ERR=/tmp/tmp.XmLVuwst21 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oKT3hDB9hr ++ cat /tmp/tmp.XmLVuwst21 ++ rm /tmp/tmp.oKT3hDB9hr /tmp/tmp.XmLVuwst21 ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for azure storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for azure storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.mhpNRBAxCU + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2232-b6dbce1b#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-26803~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + local LAST_ERR=/tmp/tmp.ydKdlk8at6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mhpNRBAxCU perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.ydKdlk8at6 + rm /tmp/tmp.mhpNRBAxCU /tmp/tmp.ydKdlk8at6 + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' ++ mktemp + local LAST_OUT=/tmp/tmp.hGvRYR39hX + /usr/bin/sed -e 's#apply:.*#apply: Never#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2232/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-26803~ + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2232-b6dbce1b#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_ERR=/tmp/tmp.whLApTZqZw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hGvRYR39hX perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.whLApTZqZw + rm /tmp/tmp.hGvRYR39hX /tmp/tmp.whLApTZqZw + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.u0qgMtVzhc +++ mktemp ++ local LAST_ERR=/tmp/tmp.GjH3l2vgSZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u0qgMtVzhc ++ cat /tmp/tmp.GjH3l2vgSZ ++ rm /tmp/tmp.u0qgMtVzhc /tmp/tmp.GjH3l2vgSZ ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.8IThowhDMx +++ mktemp ++ local LAST_ERR=/tmp/tmp.yB3h0DuYiX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8IThowhDMx ++ cat /tmp/tmp.yB3h0DuYiX ++ rm /tmp/tmp.8IThowhDMx /tmp/tmp.yB3h0DuYiX ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.MHu8qxWoOn ++ mktemp + local LAST_ERR=/tmp/tmp.PSoaOTTSbI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MHu8qxWoOn NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112103340-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-11-12-10:33:40-full Succeeded 5m15s 5m54s cron-scheduled-backup-azure-blob-2025111210399-q6fav scheduled-backup azure-blob azure://operator-testing/scheduled-backup-2025-11-12-10:39:09-full Running 25s cron-scheduled-backup-gcp-cs-2025111210379-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2025-11-12-10:37:09-full Succeeded 112s 2m25s cron-scheduled-backup-minio-20251112103525-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-11-12-10:35:25-full Succeeded 3m41s 4m9s cron-scheduled-backup-pvc-20251112103149-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112103149-q6fav-20251112103149-ac645420 Succeeded 7m11s 7m45s + cat /tmp/tmp.PSoaOTTSbI + rm /tmp/tmp.MHu8qxWoOn /tmp/tmp.PSoaOTTSbI + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.I7JU1PtyRq ++ mktemp + local LAST_ERR=/tmp/tmp.zmgiObBRr5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.I7JU1PtyRq NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-h468p 1/1 Running 0 15m pxc-client-857d976497-b9vjh 2/2 Running 0 14m scheduled-backup-proxysql-0 3/3 Running 0 14m scheduled-backup-proxysql-1 3/3 Running 0 13m scheduled-backup-pxc-0 1/1 Running 0 14m scheduled-backup-pxc-1 1/1 Running 0 12m scheduled-backup-pxc-2 1/1 Running 0 11m xb-cron-scheduled-backup-aws-s3-20251112103340-q6fav-kgb6h 0/1 Completed 0 5m55s xb-cron-scheduled-backup-azure-blob-2025111210399-q6fav-ps2s6 1/1 Running 0 26s xb-cron-scheduled-backup-gcp-cs-2025111210379-q6fav-jxqv2 0/1 Completed 0 2m26s xb-cron-scheduled-backup-minio-20251112103525-q6fav-5szdj 0/1 Completed 0 4m10s xb-cron-scheduled-backup-pvc-20251112103149-q6fav-crjq5 0/1 Completed 0 7m46s + cat /tmp/tmp.zmgiObBRr5 + rm /tmp/tmp.I7JU1PtyRq /tmp/tmp.zmgiObBRr5 + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.zqljeKI8Iq +++ mktemp ++ local LAST_ERR=/tmp/tmp.MNBX6BwLlR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zqljeKI8Iq ++ cat /tmp/tmp.MNBX6BwLlR ++ rm /tmp/tmp.zqljeKI8Iq /tmp/tmp.MNBX6BwLlR ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.9l6iK9aGuH +++ mktemp ++ local LAST_ERR=/tmp/tmp.0jTa3IdWra ++ local exit_status=0 +++ seq 0 2 ++ wc -l ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9l6iK9aGuH ++ cat /tmp/tmp.0jTa3IdWra ++ rm /tmp/tmp.9l6iK9aGuH /tmp/tmp.0jTa3IdWra ++ return 0 + [[ 0 -gt 0 ]] ++ grep pvc ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ head -n1 ++ cut -d: -f1 ++ grep Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.aJsnh3E31e +++ mktemp ++ local LAST_ERR=/tmp/tmp.92ckVVwo0r ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aJsnh3E31e ++ cat /tmp/tmp.92ckVVwo0r ++ rm /tmp/tmp.aJsnh3E31e /tmp/tmp.92ckVVwo0r ++ return 0 + FIRST_PVC_BACKUP=cron-scheduled-backup-pvc-20251112103149-q6fav ++ kubectl_bin get jobs +++ mktemp ++ local LAST_OUT=/tmp/tmp.W3VYuG8RCr ++ grep cron-scheduled-backup-pvc-20251112103149-q6fav +++ mktemp ++ local LAST_ERR=/tmp/tmp.YLqeWNgszY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ awk '{print $1}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.W3VYuG8RCr ++ cat /tmp/tmp.YLqeWNgszY ++ rm /tmp/tmp.W3VYuG8RCr /tmp/tmp.YLqeWNgszY ++ return 0 + JOB_PVC_BACKUP=xb-cron-scheduled-backup-pvc-20251112103149-q6fav ++ kubectl_bin get pods ++ awk '{print $1}' ++ grep xb-cron-scheduled-backup-pvc-20251112103149 +++ mktemp ++ local LAST_OUT=/tmp/tmp.FReuy7ilj8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.bv22MTwoez ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FReuy7ilj8 ++ cat /tmp/tmp.bv22MTwoez ++ rm /tmp/tmp.FReuy7ilj8 /tmp/tmp.bv22MTwoez ++ return 0 + POD_PVC_BACKUP=xb-cron-scheduled-backup-pvc-20251112103149-q6fav-crjq5 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded +++ mktemp ++ head -n1 ++ cut -d: -f1 ++ grep minio ++ local LAST_OUT=/tmp/tmp.53GnV9M6zZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.7GueMFSj4z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.53GnV9M6zZ ++ cat /tmp/tmp.7GueMFSj4z ++ rm /tmp/tmp.53GnV9M6zZ /tmp/tmp.7GueMFSj4z ++ return 0 + FIRST_MINIO_BACKUP=cron-scheduled-backup-minio-20251112103525-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-minio-20251112103525-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YNSaBsXoyT +++ mktemp ++ local LAST_ERR=/tmp/tmp.MmpvAsY06t ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YNSaBsXoyT ++ cat /tmp/tmp.MmpvAsY06t ++ rm /tmp/tmp.YNSaBsXoyT /tmp/tmp.MmpvAsY06t ++ return 0 + JOB_MINIO_BACKUP=xb-cron-scheduled-backup-minio-20251112103525-q6fav ++ kubectl_bin get pods ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NVghvrXaIq +++ mktemp ++ local LAST_ERR=/tmp/tmp.ivdzristF3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ grep xb-cron-scheduled-backup-minio-20251112103525 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NVghvrXaIq ++ cat /tmp/tmp.ivdzristF3 ++ rm /tmp/tmp.NVghvrXaIq /tmp/tmp.ivdzristF3 ++ return 0 + POD_MINIO_BACKUP=xb-cron-scheduled-backup-minio-20251112103525-q6fav-5szdj + '[' -z '' ']' ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ head -n1 +++ mktemp ++ grep aws ++ cut -d: -f1 ++ local LAST_OUT=/tmp/tmp.mdPhglmkFP +++ mktemp ++ local LAST_ERR=/tmp/tmp.gkgWXSx59D ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mdPhglmkFP ++ cat /tmp/tmp.gkgWXSx59D ++ rm /tmp/tmp.mdPhglmkFP /tmp/tmp.gkgWXSx59D ++ return 0 + FIRST_AWS_BACKUP=cron-scheduled-backup-aws-s3-20251112103340-q6fav ++ kubectl_bin get jobs ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QnMWc8VljL +++ mktemp ++ local LAST_ERR=/tmp/tmp.LH3c5szmS2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ grep cron-scheduled-backup-aws-s3-20251112103340-q6fav ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QnMWc8VljL ++ cat /tmp/tmp.LH3c5szmS2 ++ rm /tmp/tmp.QnMWc8VljL /tmp/tmp.LH3c5szmS2 ++ return 0 + JOB_AWS_BACKUP=xb-cron-scheduled-backup-aws-s3-20251112103340-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-aws-s3-20251112103340 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.x5uetJtodc +++ mktemp ++ local LAST_ERR=/tmp/tmp.Rb9Yp9Kn14 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.x5uetJtodc ++ cat /tmp/tmp.Rb9Yp9Kn14 ++ rm /tmp/tmp.x5uetJtodc /tmp/tmp.Rb9Yp9Kn14 ++ return 0 + POD_AWS_BACKUP=xb-cron-scheduled-backup-aws-s3-20251112103340-q6fav-kgb6h ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ head -n1 +++ mktemp ++ cut -d: -f1 ++ grep gcp ++ local LAST_OUT=/tmp/tmp.XlVEg6fd1c +++ mktemp ++ local LAST_ERR=/tmp/tmp.JqoiMWcVI9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XlVEg6fd1c ++ cat /tmp/tmp.JqoiMWcVI9 ++ rm /tmp/tmp.XlVEg6fd1c /tmp/tmp.JqoiMWcVI9 ++ return 0 + FIRST_GCP_BACKUP=cron-scheduled-backup-gcp-cs-2025111210379-q6fav ++ kubectl_bin get jobs ++ awk '{print $1}' ++ grep cron-scheduled-backup-gcp-cs-2025111210379-q6fav +++ mktemp ++ local LAST_OUT=/tmp/tmp.d4jKIJPM5J +++ mktemp ++ local LAST_ERR=/tmp/tmp.7z4TduCRqA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.d4jKIJPM5J ++ cat /tmp/tmp.7z4TduCRqA ++ rm /tmp/tmp.d4jKIJPM5J /tmp/tmp.7z4TduCRqA ++ return 0 + JOB_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-2025111210379-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-gcp-cs-2025111210379 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.im59Tmnx6B +++ mktemp ++ local LAST_ERR=/tmp/tmp.q0v45tLx4R ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.im59Tmnx6B ++ cat /tmp/tmp.q0v45tLx4R ++ rm /tmp/tmp.im59Tmnx6B /tmp/tmp.q0v45tLx4R ++ return 0 + POD_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-2025111210379-q6fav-jxqv2 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep azure ++ head -n1 +++ mktemp ++ grep Succeeded ++ local LAST_OUT=/tmp/tmp.Kr20JBx54d +++ mktemp ++ cut -d: -f1 ++ local LAST_ERR=/tmp/tmp.KCV36L5cQK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Kr20JBx54d ++ cat /tmp/tmp.KCV36L5cQK ++ rm /tmp/tmp.Kr20JBx54d /tmp/tmp.KCV36L5cQK ++ return 0 + FIRST_AZURE_BACKUP=cron-scheduled-backup-azure-blob-2025111210399-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-azure-blob-2025111210399-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WJOnIXYtkW +++ mktemp ++ local LAST_ERR=/tmp/tmp.bbe8Lwd85O ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WJOnIXYtkW ++ cat /tmp/tmp.bbe8Lwd85O ++ rm /tmp/tmp.WJOnIXYtkW /tmp/tmp.bbe8Lwd85O ++ return 0 + JOB_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-2025111210399-q6fav ++ kubectl_bin get pods ++ awk '{print $1}' ++ grep xb-cron-scheduled-backup-azure-blob-2025111210399 +++ mktemp ++ local LAST_OUT=/tmp/tmp.E4XryHkYc7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.nUKvhKkM7Z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.E4XryHkYc7 ++ cat /tmp/tmp.nUKvhKkM7Z ++ rm /tmp/tmp.E4XryHkYc7 /tmp/tmp.nUKvhKkM7Z ++ return 0 + POD_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-2025111210399-q6fav-ps2s6 ++ kubectl_bin get pxc-backup cron-scheduled-backup-aws-s3-20251112103340-q6fav -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ cut -c 6- +++ mktemp ++ local LAST_OUT=/tmp/tmp.QeWQ1admQ9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.m3w7vl2kOl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-aws-s3-20251112103340-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QeWQ1admQ9 ++ cat /tmp/tmp.m3w7vl2kOl ++ rm /tmp/tmp.QeWQ1admQ9 /tmp/tmp.m3w7vl2kOl ++ return 0 + BACKUP_DEST_AWS=operator-testing/scheduled-backup-2025-11-12-10:33:40-full ++ kubectl_bin get pxc-backup cron-scheduled-backup-gcp-cs-2025111210379-q6fav -o 'jsonpath={.status.destination}' ++ cut -c 6- ++ sed -e 's/.json$//' +++ mktemp ++ local LAST_OUT=/tmp/tmp.76ZdIxTD2g +++ mktemp ++ local LAST_ERR=/tmp/tmp.EgnQNWazFV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-gcp-cs-2025111210379-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.76ZdIxTD2g ++ cat /tmp/tmp.EgnQNWazFV ++ rm /tmp/tmp.76ZdIxTD2g /tmp/tmp.EgnQNWazFV ++ return 0 + BACKUP_DEST_GCP=operator-testing/scheduled-backup-2025-11-12-10:37:09-full ++ kubectl_bin get pxc-backup cron-scheduled-backup-azure-blob-2025111210399-q6fav -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ cut -c 9- +++ mktemp ++ local LAST_OUT=/tmp/tmp.iY4dXdFmYf +++ mktemp ++ local LAST_ERR=/tmp/tmp.FeWaiJa8IZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-azure-blob-2025111210399-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iY4dXdFmYf ++ cat /tmp/tmp.FeWaiJa8IZ ++ rm /tmp/tmp.iY4dXdFmYf /tmp/tmp.FeWaiJa8IZ ++ return 0 + BACKUP_DEST_AZURE=operator-testing/scheduled-backup-2025-11-12-10:39:09-full + desc 'Check backup existence' + set +o xtrace ----------------------------------------------------------------------------------- Check backup existence ----------------------------------------------------------------------------------- + check_backup_existence https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 aws-s3 + path=https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + storage_name=aws-s3 + retry=0 ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 0 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 1 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 2 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 3 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 4 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 5 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 6 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 7 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 8 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 9 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 10 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 11 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 12 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 13 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 14 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 15 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 16 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 17 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 18 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 19 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 20 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 21 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 22 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 23 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 24 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 25 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 26 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 27 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 28 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 29 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-10:33:40-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 30 -ge 30 ']' + echo 'max retry count 30 reached. something went wrong with operator or kubernetes cluster' max retry count 30 reached. something went wrong with operator or kubernetes cluster + echo 'Backup was not found in bucket -- aws-s3' Backup was not found in bucket -- aws-s3 + exit 1