Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/logs/scheduled-backup-8-0.log Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 + main + create_infra scheduled-backup-24766 + local ns=scheduled-backup-24766 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n scheduled-backup-29022 scheduled-backup --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.2eT18c74b0 ++ mktemp + local LAST_ERR=/tmp/tmp.s0FGVQsjMv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2eT18c74b0 perconaxtradbcluster.pxc.percona.com "scheduled-backup" deleted from scheduled-backup-29022 namespace + cat /tmp/tmp.s0FGVQsjMv + rm /tmp/tmp.2eT18c74b0 /tmp/tmp.s0FGVQsjMv + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.CmUT0bvLLn ++ mktemp + local LAST_ERR=/tmp/tmp.1Zj8bkIS7O + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CmUT0bvLLn perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-aws-s3-2025117001-q6fav" deleted from scheduled-backup-29022 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-azure-blob-202511623584-q6fav" deleted from scheduled-backup-29022 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-gcp-cs-202511623564-q6fav" deleted from scheduled-backup-29022 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-minio-202511623507-q6fav" deleted from scheduled-backup-29022 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-pvc-202511623460-q6fav" deleted from scheduled-backup-29022 namespace + cat /tmp/tmp.1Zj8bkIS7O + rm /tmp/tmp.CmUT0bvLLn /tmp/tmp.1Zj8bkIS7O + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.BHRjq47xWf ++ mktemp + local LAST_ERR=/tmp/tmp.onOFcSz3dt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BHRjq47xWf perconaxtradbclusterrestore.pxc.percona.com "pvc-202511623460-q6fav" deleted from scheduled-backup-29022 namespace + cat /tmp/tmp.onOFcSz3dt + rm /tmp/tmp.BHRjq47xWf /tmp/tmp.onOFcSz3dt + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + xargs kubectl delete ns ++ mktemp + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + local LAST_OUT=/tmp/tmp.IfNIlvQbkS ++ mktemp + local LAST_ERR=/tmp/tmp.QFCYvgn8yD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + awk '{print$1}' ++ mktemp + local LAST_OUT=/tmp/tmp.X224cguCvk ++ mktemp + local LAST_ERR=/tmp/tmp.hsgPZt0uUH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.X224cguCvk + cat /tmp/tmp.hsgPZt0uUH + rm /tmp/tmp.X224cguCvk /tmp/tmp.hsgPZt0uUH + return 0 namespace "scheduled-backup-29022" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IfNIlvQbkS namespace "pxc-operator" deleted + cat /tmp/tmp.QFCYvgn8yD + rm /tmp/tmp.IfNIlvQbkS /tmp/tmp.QFCYvgn8yD + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.Wk46j1AJW9 ++ mktemp + local LAST_ERR=/tmp/tmp.5J2A7yZxZR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Wk46j1AJW9 namespace/pxc-operator created + cat /tmp/tmp.5J2A7yZxZR + rm /tmp/tmp.Wk46j1AJW9 /tmp/tmp.5J2A7yZxZR + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Y6ZO1fcHyN +++ mktemp ++ local LAST_ERR=/tmp/tmp.yPKa6MngCs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Y6ZO1fcHyN ++ cat /tmp/tmp.yPKa6MngCs ++ rm /tmp/tmp.Y6ZO1fcHyN /tmp/tmp.yPKa6MngCs ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2154-0538614f-6-cluster7 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.zEhsQGA7eH ++ mktemp + local LAST_ERR=/tmp/tmp.cljC7aSsqO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2154-0538614f-6-cluster7 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zEhsQGA7eH Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2154-0538614f-6-cluster7" modified. + cat /tmp/tmp.cljC7aSsqO + rm /tmp/tmp.zEhsQGA7eH /tmp/tmp.cljC7aSsqO + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.WCSErPYDmY ++ mktemp + local LAST_ERR=/tmp/tmp.imbRdIp09C + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WCSErPYDmY customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.imbRdIp09C + rm /tmp/tmp.WCSErPYDmY /tmp/tmp.imbRdIp09C + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.TITIf20fk1 ++ mktemp + local LAST_ERR=/tmp/tmp.u3Iss3x8aG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TITIf20fk1 clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.u3Iss3x8aG + rm /tmp/tmp.TITIf20fk1 /tmp/tmp.u3Iss3x8aG + return 0 + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2154-0538614f^' + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - ++ mktemp + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/deploy/cw-operator.yaml + local LAST_OUT=/tmp/tmp.imYNwzoRGn ++ mktemp + local LAST_ERR=/tmp/tmp.LJ3JhdkNrB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.imYNwzoRGn deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.LJ3JhdkNrB + rm /tmp/tmp.imYNwzoRGn /tmp/tmp.LJ3JhdkNrB + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.h94aF7mbje ++ mktemp + local LAST_ERR=/tmp/tmp.2yY7W53pMO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.h94aF7mbje pod/percona-xtradb-cluster-operator-c799c8d46-nl2ds condition met + cat /tmp/tmp.2yY7W53pMO + rm /tmp/tmp.h94aF7mbje /tmp/tmp.2yY7W53pMO + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.OPJAINmexL +++ mktemp ++ local LAST_ERR=/tmp/tmp.rOKHXpiouD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OPJAINmexL ++ cat /tmp/tmp.rOKHXpiouD ++ rm /tmp/tmp.OPJAINmexL /tmp/tmp.rOKHXpiouD ++ return 0 + wait_pod percona-xtradb-cluster-operator-c799c8d46-nl2ds 480 pxc-operator + local pod=percona-xtradb-cluster-operator-c799c8d46-nl2ds + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-c799c8d46-nl2ds ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-c799c8d46-nl2ds condition met waiting for pod/percona-xtradb-cluster-operator-c799c8d46-nl2ds to become Ready.Ok + sleep 3 + create_namespace scheduled-backup-24766 + local namespace=scheduled-backup-24766 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces scheduled-backup-24766' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces scheduled-backup-24766 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace scheduled-backup-24766 + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.34egfonsQN ++ mktemp + local LAST_ERR=/tmp/tmp.hqBYDjTOf5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-24766 ++ mktemp + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + local LAST_OUT=/tmp/tmp.iwnDrVg3xi ++ mktemp + local LAST_ERR=/tmp/tmp.UPNwjDa9Zo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-24766 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iwnDrVg3xi + cat /tmp/tmp.UPNwjDa9Zo + rm /tmp/tmp.iwnDrVg3xi /tmp/tmp.UPNwjDa9Zo + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-24766 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.34egfonsQN + cat /tmp/tmp.hqBYDjTOf5 Error from server (NotFound): namespaces "scheduled-backup-24766" not found + rm /tmp/tmp.34egfonsQN /tmp/tmp.hqBYDjTOf5 + return 1 + : + wait_for_delete namespace/scheduled-backup-24766 + local res=namespace/scheduled-backup-24766 + echo -n 'waiting for namespace/scheduled-backup-24766 to be deleted' waiting for namespace/scheduled-backup-24766 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "scheduled-backup-24766" not found + desc 'create namespace scheduled-backup-24766' + set +o xtrace ----------------------------------------------------------------------------------- create namespace scheduled-backup-24766 ----------------------------------------------------------------------------------- + kubectl_bin create namespace scheduled-backup-24766 ++ mktemp + local LAST_OUT=/tmp/tmp.lLHlQmf4j3 ++ mktemp + local LAST_ERR=/tmp/tmp.k7V78bYb2J + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace scheduled-backup-24766 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lLHlQmf4j3 namespace/scheduled-backup-24766 created + cat /tmp/tmp.k7V78bYb2J + rm /tmp/tmp.lLHlQmf4j3 /tmp/tmp.k7V78bYb2J + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.KABe9Efukd +++ mktemp ++ local LAST_ERR=/tmp/tmp.ML4XZ2ThXc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KABe9Efukd ++ cat /tmp/tmp.ML4XZ2ThXc ++ rm /tmp/tmp.KABe9Efukd /tmp/tmp.ML4XZ2ThXc ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2154-0538614f-6-cluster7 --namespace=scheduled-backup-24766 ++ mktemp + local LAST_OUT=/tmp/tmp.LmecUcPCKF ++ mktemp + local LAST_ERR=/tmp/tmp.QDPaAbert0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2154-0538614f-6-cluster7 --namespace=scheduled-backup-24766 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LmecUcPCKF Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2154-0538614f-6-cluster7" modified. + cat /tmp/tmp.QDPaAbert0 + rm /tmp/tmp.LmecUcPCKF /tmp/tmp.QDPaAbert0 + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.6l6C8KvNgo ++ mktemp + local LAST_ERR=/tmp/tmp.xYE9gkCquS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6l6C8KvNgo secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.xYE9gkCquS + rm /tmp/tmp.6l6C8KvNgo /tmp/tmp.xYE9gkCquS + return 0 + start_minio + deploy_helm scheduled-backup-24766 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "hashicorp" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Fri Nov 7 00:30:05 2025 NAMESPACE: scheduled-backup-24766 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.scheduled-backup-24766.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-24766 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-24766 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-24766 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-24766 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PehWUTYQdR +++ mktemp ++ local LAST_ERR=/tmp/tmp.xBiAHeJoFS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PehWUTYQdR ++ cat /tmp/tmp.xBiAHeJoFS ++ rm /tmp/tmp.PehWUTYQdR /tmp/tmp.xBiAHeJoFS ++ return 0 + MINIO_POD=minio-service-55fcc5d75f-4vrrx + wait_pod minio-service-55fcc5d75f-4vrrx + local pod=minio-service-55fcc5d75f-4vrrx + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo minio-service-55fcc5d75f-4vrrx ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minio-service-55fcc5d75f-4vrrx condition met waiting for pod/minio-service-55fcc5d75f-4vrrx to become Ready.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.S9916RGdt3 ++ mktemp + local LAST_ERR=/tmp/tmp.eDrByBiHjm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.S9916RGdt3 make_bucket: operator-testing pod "aws-cli" deleted from scheduled-backup-24766 namespace + cat /tmp/tmp.eDrByBiHjm All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_scheduled-backup-24766 + rm /tmp/tmp.S9916RGdt3 /tmp/tmp.eDrByBiHjm + return 0 + cluster=scheduled-backup + cat - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.xtx3Yv4Dta ++ mktemp + local LAST_ERR=/tmp/tmp.0wfR3HgMDT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xtx3Yv4Dta priorityclass.scheduling.k8s.io/high-priority configured + cat /tmp/tmp.0wfR3HgMDT + rm /tmp/tmp.xtx3Yv4Dta /tmp/tmp.0wfR3HgMDT + return 0 + spinup_pxc scheduled-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local cluster=scheduled-backup + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.uQURHjemot ++ mktemp + local LAST_ERR=/tmp/tmp.14uIuZRTEZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uQURHjemot secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.14uIuZRTEZ + rm /tmp/tmp.uQURHjemot /tmp/tmp.14uIuZRTEZ + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.bJn64963bI ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_ERR=/tmp/tmp.fcAVE6ekwn + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-24766~ ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2154-0538614f#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bJn64963bI deployment.apps/pxc-client created + cat /tmp/tmp.fcAVE6ekwn + rm /tmp/tmp.bJn64963bI /tmp/tmp.fcAVE6ekwn + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml ++ mktemp + local LAST_OUT=/tmp/tmp.sOFrs94MbV ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.enaxjFEK97 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-24766~ + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2154-0538614f#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sOFrs94MbV perconaxtradbcluster.pxc.percona.com/scheduled-backup created + cat /tmp/tmp.enaxjFEK97 + rm /tmp/tmp.sOFrs94MbV /tmp/tmp.enaxjFEK97 + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy scheduled-backup ++ local target_cluster=scheduled-backup +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VvHmj650mp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.CEUG7nWY0h +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.VvHmj650mp +++ cat /tmp/tmp.CEUG7nWY0h +++ rm /tmp/tmp.VvHmj650mp /tmp/tmp.CEUG7nWY0h +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.H4oevrotxs ++++ mktemp +++ local LAST_ERR=/tmp/tmp.r35p9eOlal +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.H4oevrotxs +++ cat /tmp/tmp.r35p9eOlal +++ rm /tmp/tmp.H4oevrotxs /tmp/tmp.r35p9eOlal +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scheduled-backup-proxysql ++ return + local proxy=scheduled-backup-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-24766 ++ mktemp + local LAST_OUT=/tmp/tmp.VcE8hrkIXJ ++ mktemp + local LAST_ERR=/tmp/tmp.ytnTpjKSG3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-24766 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-24766 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-24766 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.VcE8hrkIXJ + cat /tmp/tmp.ytnTpjKSG3 error: no matching resources found + rm /tmp/tmp.VcE8hrkIXJ /tmp/tmp.ytnTpjKSG3 + return 1 + true + wait_for_running scheduled-backup-proxysql 1 + local name=scheduled-backup-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-proxysql-0 480 + local pod=scheduled-backup-proxysql-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/scheduled-backup-proxysql-0 condition met waiting for pod/scheduled-backup-proxysql-0 to become Ready.Ok + wait_for_running scheduled-backup-pxc 3 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 480 + local pod=scheduled-backup-pxc-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-0 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 480 + local pod=scheduled-backup-pxc-1 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 480 + local pod=scheduled-backup-pxc-2 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.uMggjhnG5v +++ mktemp ++ local LAST_ERR=/tmp/tmp.3yXCaPpWRU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uMggjhnG5v ++ cat /tmp/tmp.3yXCaPpWRU ++ rm /tmp/tmp.uMggjhnG5v /tmp/tmp.3yXCaPpWRU ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EiGpKB9L17 +++ mktemp ++ local LAST_ERR=/tmp/tmp.b4BGXk3ecv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EiGpKB9L17 ++ cat /tmp/tmp.b4BGXk3ecv ++ rm /tmp/tmp.EiGpKB9L17 /tmp/tmp.b4BGXk3ecv ++ return 0 + client_pod=pxc-client-59944c5bbf-j2gl5 + wait_pod pxc-client-59944c5bbf-j2gl5 + local pod=pxc-client-59944c5bbf-j2gl5 + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-j2gl5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-j2gl5 condition met waiting for pod/pxc-client-59944c5bbf-j2gl5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aavAaIP8uc +++ mktemp ++ local LAST_ERR=/tmp/tmp.DFRzMbGMO3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aavAaIP8uc ++ cat /tmp/tmp.DFRzMbGMO3 ++ rm /tmp/tmp.aavAaIP8uc /tmp/tmp.DFRzMbGMO3 ++ return 0 + client_pod=pxc-client-59944c5bbf-j2gl5 + wait_pod pxc-client-59944c5bbf-j2gl5 + local pod=pxc-client-59944c5bbf-j2gl5 + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-j2gl5 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-j2gl5 condition met waiting for pod/pxc-client-59944c5bbf-j2gl5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.56QYLgGKjS +++ mktemp ++ local LAST_ERR=/tmp/tmp.EyoEnOeo9F ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.56QYLgGKjS ++ cat /tmp/tmp.EyoEnOeo9F ++ rm /tmp/tmp.56QYLgGKjS /tmp/tmp.EyoEnOeo9F ++ return 0 + client_pod=pxc-client-59944c5bbf-j2gl5 + wait_pod pxc-client-59944c5bbf-j2gl5 + local pod=pxc-client-59944c5bbf-j2gl5 + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-j2gl5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-j2gl5 condition met waiting for pod/pxc-client-59944c5bbf-j2gl5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.pjuSwpbVyK/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.pjuSwpbVyK/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q9sqYaLUQC +++ mktemp ++ local LAST_ERR=/tmp/tmp.fXscpFRdqw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.q9sqYaLUQC ++ cat /tmp/tmp.fXscpFRdqw ++ rm /tmp/tmp.q9sqYaLUQC /tmp/tmp.fXscpFRdqw ++ return 0 + client_pod=pxc-client-59944c5bbf-j2gl5 + wait_pod pxc-client-59944c5bbf-j2gl5 + local pod=pxc-client-59944c5bbf-j2gl5 + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-j2gl5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-j2gl5 condition met waiting for pod/pxc-client-59944c5bbf-j2gl5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.pjuSwpbVyK/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.pjuSwpbVyK/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.G1BDXnf5Ta +++ mktemp ++ local LAST_ERR=/tmp/tmp.NfVBxqyL2F ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.G1BDXnf5Ta ++ cat /tmp/tmp.NfVBxqyL2F ++ rm /tmp/tmp.G1BDXnf5Ta /tmp/tmp.NfVBxqyL2F ++ return 0 + client_pod=pxc-client-59944c5bbf-j2gl5 + wait_pod pxc-client-59944c5bbf-j2gl5 + local pod=pxc-client-59944c5bbf-j2gl5 + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-j2gl5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-j2gl5 condition met waiting for pod/pxc-client-59944c5bbf-j2gl5 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.pjuSwpbVyK/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.pjuSwpbVyK/select-1.sql ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BAGKFlcc6k +++ mktemp ++ local LAST_ERR=/tmp/tmp.cSFg5nWoko ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BAGKFlcc6k ++ cat /tmp/tmp.cSFg5nWoko Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.BAGKFlcc6k /tmp/tmp.cSFg5nWoko ++ return 0 + '[' '' ']' + sleep 20 + desc 'add backups schedule for pvc storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for pvc storage ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.zM6fiztSXK +++ mktemp ++ local LAST_ERR=/tmp/tmp.uhzgu5gK12 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zM6fiztSXK ++ cat /tmp/tmp.uhzgu5gK12 ++ rm /tmp/tmp.zM6fiztSXK /tmp/tmp.uhzgu5gK12 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2154-0538614f-6-cluster7 --namespace=scheduled-backup-24766 ++ mktemp + local LAST_OUT=/tmp/tmp.cDWWDBvEpP ++ mktemp + local LAST_ERR=/tmp/tmp.Cmh9uiotmg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2154-0538614f-6-cluster7 --namespace=scheduled-backup-24766 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cDWWDBvEpP Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2154-0538614f-6-cluster7" modified. + cat /tmp/tmp.Cmh9uiotmg + rm /tmp/tmp.cDWWDBvEpP /tmp/tmp.Cmh9uiotmg + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2154-0538614f#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-24766~ + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_OUT=/tmp/tmp.R9zeHtbhzW ++ mktemp + local LAST_ERR=/tmp/tmp.t4RotHFHfJ + local exit_status=0 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.R9zeHtbhzW perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.t4RotHFHfJ + rm /tmp/tmp.R9zeHtbhzW /tmp/tmp.t4RotHFHfJ + return 0 + label_node ++ kubectl_bin get nodes --no-headers=true ++ grep -v master ++ head -n1 +++ mktemp ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.iYcXiz8ojU +++ mktemp ++ local LAST_ERR=/tmp/tmp.8TzBxEI0vH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get nodes --no-headers=true ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iYcXiz8ojU ++ cat /tmp/tmp.8TzBxEI0vH ++ rm /tmp/tmp.iYcXiz8ojU /tmp/tmp.8TzBxEI0vH ++ return 0 + LABELED_NODE=gke-jen-pxc-2154-0538614-default-pool-74f82051-3l2l + kubectl_bin label nodes gke-jen-pxc-2154-0538614-default-pool-74f82051-3l2l backupWorker=True --overwrite ++ mktemp + local LAST_OUT=/tmp/tmp.r6DM7eFAg4 ++ mktemp + local LAST_ERR=/tmp/tmp.CkKNNdoBCr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label nodes gke-jen-pxc-2154-0538614-default-pool-74f82051-3l2l backupWorker=True --overwrite + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.r6DM7eFAg4 node/gke-jen-pxc-2154-0538614-default-pool-74f82051-3l2l not labeled + cat /tmp/tmp.CkKNNdoBCr + rm /tmp/tmp.r6DM7eFAg4 /tmp/tmp.CkKNNdoBCr + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.njTC1LZFSw + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2154-0538614f#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-24766~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_ERR=/tmp/tmp.nOVeyAVTcT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.njTC1LZFSw perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.nOVeyAVTcT + rm /tmp/tmp.njTC1LZFSw /tmp/tmp.nOVeyAVTcT + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.zrCILJQXmk +++ mktemp ++ local LAST_ERR=/tmp/tmp.QmHQlUtZ4X ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zrCILJQXmk ++ cat /tmp/tmp.QmHQlUtZ4X ++ rm /tmp/tmp.zrCILJQXmk /tmp/tmp.QmHQlUtZ4X ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ grep :Failed ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.g0DwePKgfP +++ mktemp ++ local LAST_ERR=/tmp/tmp.MQTD851a57 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.g0DwePKgfP ++ cat /tmp/tmp.MQTD851a57 ++ rm /tmp/tmp.g0DwePKgfP /tmp/tmp.MQTD851a57 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.ScQybkwWom ++ mktemp + local LAST_ERR=/tmp/tmp.ItoosFeoSF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ScQybkwWom NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-202511703730-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202511703730-q6fav Running 19s + cat /tmp/tmp.ItoosFeoSF + rm /tmp/tmp.ScQybkwWom /tmp/tmp.ItoosFeoSF + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.9KiXAMKhDw ++ mktemp + local LAST_ERR=/tmp/tmp.Sxl57UlMfy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9KiXAMKhDw NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-4vrrx 1/1 Running 0 7m42s pxc-client-59944c5bbf-j2gl5 2/2 Running 0 6m30s scheduled-backup-proxysql-0 3/3 Running 0 6m27s scheduled-backup-proxysql-1 3/3 Running 0 6m8s scheduled-backup-pxc-0 1/1 Running 0 6m27s scheduled-backup-pxc-1 1/1 Running 0 5m14s scheduled-backup-pxc-2 1/1 Running 0 4m xb-cron-scheduled-backup-pvc-202511703730-q6fav-khk2t 1/1 Running 0 20s + cat /tmp/tmp.Sxl57UlMfy + rm /tmp/tmp.9KiXAMKhDw /tmp/tmp.Sxl57UlMfy + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Qgzg1VMCiz +++ mktemp ++ local LAST_ERR=/tmp/tmp.JIAv6clcvE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Qgzg1VMCiz ++ cat /tmp/tmp.JIAv6clcvE ++ rm /tmp/tmp.Qgzg1VMCiz /tmp/tmp.JIAv6clcvE ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.AHEGQ4FOOi +++ mktemp ++ local LAST_ERR=/tmp/tmp.UvQzyKG3eA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AHEGQ4FOOi ++ cat /tmp/tmp.UvQzyKG3eA ++ rm /tmp/tmp.AHEGQ4FOOi /tmp/tmp.UvQzyKG3eA ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.di20KRpXPK + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2154-0538614f#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-24766~ + local LAST_ERR=/tmp/tmp.lktUKrrGTH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.di20KRpXPK perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.lktUKrrGTH + rm /tmp/tmp.di20KRpXPK /tmp/tmp.lktUKrrGTH + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.YZMylJ4xxQ + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-24766~ + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2154-0538614f#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_ERR=/tmp/tmp.UcHtNHXK8f + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YZMylJ4xxQ perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.UcHtNHXK8f + rm /tmp/tmp.YZMylJ4xxQ /tmp/tmp.UcHtNHXK8f + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.RSystWQor4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ODwJbXeWNe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RSystWQor4 ++ cat /tmp/tmp.ODwJbXeWNe ++ rm /tmp/tmp.RSystWQor4 /tmp/tmp.ODwJbXeWNe ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.eBX01eRiDB +++ mktemp ++ local LAST_ERR=/tmp/tmp.P24k7fB0bL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eBX01eRiDB ++ cat /tmp/tmp.P24k7fB0bL ++ rm /tmp/tmp.eBX01eRiDB /tmp/tmp.P24k7fB0bL ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for minio storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for minio storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.PjVG6k6a0Q + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2154-0538614f#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.CfXbhfJDUT + local exit_status=0 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-24766~ ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PjVG6k6a0Q perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.CfXbhfJDUT + rm /tmp/tmp.PjVG6k6a0Q /tmp/tmp.CfXbhfJDUT + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2154-0538614f#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_OUT=/tmp/tmp.h7joO1uFjZ + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-24766~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.uSCL6zH5gj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.h7joO1uFjZ perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.uSCL6zH5gj + rm /tmp/tmp.h7joO1uFjZ /tmp/tmp.uSCL6zH5gj + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.gfxuYdCDBL +++ mktemp ++ local LAST_ERR=/tmp/tmp.Cqf4kwdNGr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gfxuYdCDBL ++ cat /tmp/tmp.Cqf4kwdNGr ++ rm /tmp/tmp.gfxuYdCDBL /tmp/tmp.Cqf4kwdNGr ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.LBEgnXRMgt +++ mktemp ++ local LAST_ERR=/tmp/tmp.lUhdiMOCoJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LBEgnXRMgt ++ cat /tmp/tmp.lUhdiMOCoJ ++ rm /tmp/tmp.LBEgnXRMgt /tmp/tmp.lUhdiMOCoJ ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.GF5MCVpyjY ++ mktemp + local LAST_ERR=/tmp/tmp.58L1lXpUG3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GF5MCVpyjY NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-minio-202511704019-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-11-07-00:40:19-full Running 14s cron-scheduled-backup-pvc-202511703730-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202511703730-q6fav Succeeded 2m25s 3m3s + cat /tmp/tmp.58L1lXpUG3 + rm /tmp/tmp.GF5MCVpyjY /tmp/tmp.58L1lXpUG3 + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.UNFzLgNNW0 ++ mktemp + local LAST_ERR=/tmp/tmp.TlM9NKmMpv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UNFzLgNNW0 NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-4vrrx 1/1 Running 0 10m pxc-client-59944c5bbf-j2gl5 2/2 Running 0 9m13s scheduled-backup-proxysql-0 3/3 Running 0 9m10s scheduled-backup-proxysql-1 3/3 Running 0 8m51s scheduled-backup-pxc-0 1/1 Running 0 9m10s scheduled-backup-pxc-1 1/1 Running 0 7m57s scheduled-backup-pxc-2 1/1 Running 0 6m43s xb-cron-scheduled-backup-minio-202511704019-q6fav-h8ftn 1/1 Running 0 14s xb-cron-scheduled-backup-pvc-202511703730-q6fav-khk2t 0/1 Completed 0 3m3s + cat /tmp/tmp.TlM9NKmMpv + rm /tmp/tmp.UNFzLgNNW0 /tmp/tmp.TlM9NKmMpv + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.oVlHggHBjw +++ mktemp ++ local LAST_ERR=/tmp/tmp.1I3ozMQhww ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oVlHggHBjw ++ cat /tmp/tmp.1I3ozMQhww ++ rm /tmp/tmp.oVlHggHBjw /tmp/tmp.1I3ozMQhww ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.490oCl5143 +++ mktemp ++ local LAST_ERR=/tmp/tmp.QJGtCzTMD7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.490oCl5143 ++ cat /tmp/tmp.QJGtCzTMD7 ++ rm /tmp/tmp.490oCl5143 /tmp/tmp.QJGtCzTMD7 ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for gcs storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for gcs storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2154-0538614f#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_OUT=/tmp/tmp.lWstkl57SD + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-24766~ + local LAST_ERR=/tmp/tmp.gBISmg1Kqd + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lWstkl57SD perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.gBISmg1Kqd + rm /tmp/tmp.lWstkl57SD /tmp/tmp.gBISmg1Kqd + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.4GS2h8Zqtg + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2154-0538614f#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-24766~ + local LAST_ERR=/tmp/tmp.WnC8gZCow5 + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4GS2h8Zqtg perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.WnC8gZCow5 + rm /tmp/tmp.4GS2h8Zqtg /tmp/tmp.WnC8gZCow5 + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.ScoDadlQEE +++ mktemp ++ local LAST_ERR=/tmp/tmp.ibNmFR4iKJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ScoDadlQEE ++ cat /tmp/tmp.ibNmFR4iKJ ++ rm /tmp/tmp.ScoDadlQEE /tmp/tmp.ibNmFR4iKJ ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.vBOpnX9sEZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.CE3Fy45Mh7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vBOpnX9sEZ ++ cat /tmp/tmp.CE3Fy45Mh7 ++ rm /tmp/tmp.vBOpnX9sEZ /tmp/tmp.CE3Fy45Mh7 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.vRoujCibuQ ++ mktemp + local LAST_ERR=/tmp/tmp.LgB9Ed3S2j + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vRoujCibuQ NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-gcp-cs-202511704157-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2025-11-07-00:41:57-full Running 17s cron-scheduled-backup-minio-202511704019-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-11-07-00:40:19-full Succeeded 87s 115s cron-scheduled-backup-pvc-202511703730-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202511703730-q6fav Succeeded 4m6s 4m44s + cat /tmp/tmp.LgB9Ed3S2j + rm /tmp/tmp.vRoujCibuQ /tmp/tmp.LgB9Ed3S2j + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.5dquPWdKy3 ++ mktemp + local LAST_ERR=/tmp/tmp.ViclXmRzli + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5dquPWdKy3 NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-4vrrx 1/1 Running 0 12m pxc-client-59944c5bbf-j2gl5 2/2 Running 0 10m scheduled-backup-proxysql-0 3/3 Running 0 10m scheduled-backup-proxysql-1 3/3 Running 0 10m scheduled-backup-pxc-0 1/1 Running 0 10m scheduled-backup-pxc-1 1/1 Running 0 9m38s scheduled-backup-pxc-2 1/1 Running 0 8m24s xb-cron-scheduled-backup-gcp-cs-202511704157-q6fav-kh6wk 1/1 Running 0 17s xb-cron-scheduled-backup-minio-202511704019-q6fav-h8ftn 0/1 Completed 0 115s xb-cron-scheduled-backup-pvc-202511703730-q6fav-khk2t 0/1 Completed 0 4m44s + cat /tmp/tmp.ViclXmRzli + rm /tmp/tmp.5dquPWdKy3 /tmp/tmp.ViclXmRzli + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.8NSgC5pHkf +++ mktemp ++ local LAST_ERR=/tmp/tmp.VnUGRYakua ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8NSgC5pHkf ++ cat /tmp/tmp.VnUGRYakua ++ rm /tmp/tmp.8NSgC5pHkf /tmp/tmp.VnUGRYakua ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.j5e2Tg8zQl +++ mktemp ++ local LAST_ERR=/tmp/tmp.iZQ6GijxJv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.j5e2Tg8zQl ++ cat /tmp/tmp.iZQ6GijxJv ++ rm /tmp/tmp.j5e2Tg8zQl /tmp/tmp.iZQ6GijxJv ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for azure storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for azure storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_OUT=/tmp/tmp.3BnQbbolQO ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2154-0538614f#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_ERR=/tmp/tmp.pPmid1htc8 + local exit_status=0 + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-24766~ + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3BnQbbolQO perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.pPmid1htc8 + rm /tmp/tmp.3BnQbbolQO /tmp/tmp.pPmid1htc8 + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2154/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2154-0538614f#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_OUT=/tmp/tmp.ELB4moi1JA ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-24766~ + local LAST_ERR=/tmp/tmp.kPmMh09wC6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ELB4moi1JA perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.kPmMh09wC6 + rm /tmp/tmp.ELB4moi1JA /tmp/tmp.kPmMh09wC6 + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.ISvhGCWMd9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.XJgoXasUpW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ISvhGCWMd9 ++ cat /tmp/tmp.XJgoXasUpW ++ rm /tmp/tmp.ISvhGCWMd9 /tmp/tmp.XJgoXasUpW ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.WNkICUY9jW +++ mktemp ++ local LAST_ERR=/tmp/tmp.CGdWDTeEgO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WNkICUY9jW ++ cat /tmp/tmp.CGdWDTeEgO ++ rm /tmp/tmp.WNkICUY9jW /tmp/tmp.CGdWDTeEgO ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.GQLHwwUjU8 ++ mktemp + local LAST_ERR=/tmp/tmp.GxWd99vLny + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GQLHwwUjU8 NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-azure-blob-202511704339-q6fav scheduled-backup azure-blob azure://operator-testing/scheduled-backup-2025-11-07-00:43:39-full Running 12s cron-scheduled-backup-gcp-cs-202511704157-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2025-11-07-00:41:57-full Succeeded 83s 114s cron-scheduled-backup-minio-202511704019-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-11-07-00:40:19-full Succeeded 3m4s 3m32s cron-scheduled-backup-pvc-202511703730-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202511703730-q6fav Succeeded 5m43s 6m21s + cat /tmp/tmp.GxWd99vLny + rm /tmp/tmp.GQLHwwUjU8 /tmp/tmp.GxWd99vLny + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.l6M4Ef49Vg ++ mktemp + local LAST_ERR=/tmp/tmp.WtK0RZACOn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.l6M4Ef49Vg NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-4vrrx 1/1 Running 0 13m pxc-client-59944c5bbf-j2gl5 2/2 Running 0 12m scheduled-backup-proxysql-0 3/3 Running 0 12m scheduled-backup-proxysql-1 3/3 Running 0 12m scheduled-backup-pxc-0 1/1 Running 0 12m scheduled-backup-pxc-1 1/1 Running 0 11m scheduled-backup-pxc-2 1/1 Running 0 10m xb-cron-scheduled-backup-azure-blob-202511704339-q6fav-c5g58 1/1 Running 0 13s xb-cron-scheduled-backup-gcp-cs-202511704157-q6fav-kh6wk 0/1 Completed 0 115s xb-cron-scheduled-backup-minio-202511704019-q6fav-h8ftn 0/1 Completed 0 3m33s xb-cron-scheduled-backup-pvc-202511703730-q6fav-khk2t 0/1 Completed 0 6m22s + cat /tmp/tmp.WtK0RZACOn + rm /tmp/tmp.l6M4Ef49Vg /tmp/tmp.WtK0RZACOn + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.KBbDJEVyMF +++ mktemp ++ local LAST_ERR=/tmp/tmp.Iwlw8Md37Y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KBbDJEVyMF ++ cat /tmp/tmp.Iwlw8Md37Y ++ rm /tmp/tmp.KBbDJEVyMF /tmp/tmp.Iwlw8Md37Y ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.IQDsLcBI2f +++ mktemp ++ local LAST_ERR=/tmp/tmp.YFNP6xOKTX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IQDsLcBI2f ++ cat /tmp/tmp.YFNP6xOKTX ++ rm /tmp/tmp.IQDsLcBI2f /tmp/tmp.YFNP6xOKTX ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.y0e1v141K6 ++ mktemp + local LAST_ERR=/tmp/tmp.ZvYKFRPwMd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.y0e1v141K6 NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-azure-blob-202511704339-q6fav scheduled-backup azure-blob azure://operator-testing/scheduled-backup-2025-11-07-00:43:39-full Succeeded 13s 43s cron-scheduled-backup-gcp-cs-202511704157-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2025-11-07-00:41:57-full Succeeded 114s 2m25s cron-scheduled-backup-minio-202511704019-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-11-07-00:40:19-full Succeeded 3m35s 4m3s cron-scheduled-backup-pvc-202511703730-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202511703730-q6fav Succeeded 6m14s 6m52s + cat /tmp/tmp.ZvYKFRPwMd + rm /tmp/tmp.y0e1v141K6 /tmp/tmp.ZvYKFRPwMd + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.Oo9Uw6pwFp ++ mktemp + local LAST_ERR=/tmp/tmp.4IbEwVruvB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Oo9Uw6pwFp NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-4vrrx 1/1 Running 0 14m pxc-client-59944c5bbf-j2gl5 2/2 Running 0 13m scheduled-backup-proxysql-0 3/3 Running 0 12m scheduled-backup-proxysql-1 3/3 Running 0 12m scheduled-backup-pxc-0 1/1 Running 0 12m scheduled-backup-pxc-1 1/1 Running 0 11m scheduled-backup-pxc-2 1/1 Running 0 10m xb-cron-scheduled-backup-azure-blob-202511704339-q6fav-c5g58 0/1 Completed 0 43s xb-cron-scheduled-backup-gcp-cs-202511704157-q6fav-kh6wk 0/1 Completed 0 2m25s xb-cron-scheduled-backup-minio-202511704019-q6fav-h8ftn 0/1 Completed 0 4m3s xb-cron-scheduled-backup-pvc-202511703730-q6fav-khk2t 0/1 Completed 0 6m52s + cat /tmp/tmp.4IbEwVruvB + rm /tmp/tmp.Oo9Uw6pwFp /tmp/tmp.4IbEwVruvB + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Rt1dw6hoSh +++ mktemp ++ local LAST_ERR=/tmp/tmp.UVJrXee2wf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Rt1dw6hoSh ++ cat /tmp/tmp.UVJrXee2wf ++ rm /tmp/tmp.Rt1dw6hoSh /tmp/tmp.UVJrXee2wf ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.xRQew6rLi9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.sOwqUC8bYe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xRQew6rLi9 ++ cat /tmp/tmp.sOwqUC8bYe ++ rm /tmp/tmp.xRQew6rLi9 /tmp/tmp.sOwqUC8bYe ++ return 0 + [[ 0 -gt 0 ]] ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ grep pvc +++ mktemp ++ cut -d: -f1 ++ head -n1 ++ local LAST_OUT=/tmp/tmp.guAId6YEJv +++ mktemp ++ local LAST_ERR=/tmp/tmp.MPU45C29y4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.guAId6YEJv ++ cat /tmp/tmp.MPU45C29y4 ++ rm /tmp/tmp.guAId6YEJv /tmp/tmp.MPU45C29y4 ++ return 0 + FIRST_PVC_BACKUP=cron-scheduled-backup-pvc-202511703730-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-pvc-202511703730-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.x7A7fqGJxw +++ mktemp ++ local LAST_ERR=/tmp/tmp.yYPMe3wNdJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.x7A7fqGJxw ++ cat /tmp/tmp.yYPMe3wNdJ ++ rm /tmp/tmp.x7A7fqGJxw /tmp/tmp.yYPMe3wNdJ ++ return 0 + JOB_PVC_BACKUP=xb-cron-scheduled-backup-pvc-202511703730-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-pvc-202511703730 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.R8ElShQOE9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.L2RPsagUBY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.R8ElShQOE9 ++ cat /tmp/tmp.L2RPsagUBY ++ rm /tmp/tmp.R8ElShQOE9 /tmp/tmp.L2RPsagUBY ++ return 0 + POD_PVC_BACKUP=xb-cron-scheduled-backup-pvc-202511703730-q6fav-khk2t ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' +++ mktemp ++ cut -d: -f1 ++ head -n1 ++ grep minio ++ grep Succeeded ++ local LAST_OUT=/tmp/tmp.wlyiBYE019 +++ mktemp ++ local LAST_ERR=/tmp/tmp.dA1K7O47sh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wlyiBYE019 ++ cat /tmp/tmp.dA1K7O47sh ++ rm /tmp/tmp.wlyiBYE019 /tmp/tmp.dA1K7O47sh ++ return 0 + FIRST_MINIO_BACKUP=cron-scheduled-backup-minio-202511704019-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-minio-202511704019-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1l3WIsyEUf +++ mktemp ++ local LAST_ERR=/tmp/tmp.esFMqbuWeW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1l3WIsyEUf ++ cat /tmp/tmp.esFMqbuWeW ++ rm /tmp/tmp.1l3WIsyEUf /tmp/tmp.esFMqbuWeW ++ return 0 + JOB_MINIO_BACKUP=xb-cron-scheduled-backup-minio-202511704019-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-minio-202511704019 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BVJQtMqI6n +++ mktemp ++ local LAST_ERR=/tmp/tmp.Wa5QKsVGoc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BVJQtMqI6n ++ cat /tmp/tmp.Wa5QKsVGoc ++ rm /tmp/tmp.BVJQtMqI6n /tmp/tmp.Wa5QKsVGoc ++ return 0 + POD_MINIO_BACKUP=xb-cron-scheduled-backup-minio-202511704019-q6fav-h8ftn + '[' -z '' ']' ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ head -n1 +++ mktemp ++ cut -d: -f1 ++ grep aws ++ local LAST_OUT=/tmp/tmp.thmA8x2DJM +++ mktemp ++ local LAST_ERR=/tmp/tmp.pkfWIhWjLK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.thmA8x2DJM ++ cat /tmp/tmp.pkfWIhWjLK ++ rm /tmp/tmp.thmA8x2DJM /tmp/tmp.pkfWIhWjLK ++ return 0 + FIRST_AWS_BACKUP= ++ kubectl_bin get jobs ++ grep ++ awk '{print $1}' +++ mktemp Usage: grep [OPTION]... PATTERN [FILE]... Try 'grep --help' for more information. ++ local LAST_OUT=/tmp/tmp.uLgAI8gdaY +++ mktemp ++ local LAST_ERR=/tmp/tmp.3KPZmrwYQY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uLgAI8gdaY + JOB_AWS_BACKUP= ++ kubectl_bin get pods ++ grep ++ awk '{print $1}' +++ mktemp Usage: grep [OPTION]... PATTERN [FILE]... Try 'grep --help' for more information. ++ local LAST_OUT=/tmp/tmp.2i4UGlxU87 +++ mktemp ++ local LAST_ERR=/tmp/tmp.5KL5Ieycml ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2i4UGlxU87 + POD_AWS_BACKUP= ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ head -n1 +++ mktemp ++ grep gcp ++ cut -d: -f1 ++ local LAST_OUT=/tmp/tmp.V7o22rA34g +++ mktemp ++ local LAST_ERR=/tmp/tmp.I5cr0cppLK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.V7o22rA34g ++ cat /tmp/tmp.I5cr0cppLK ++ rm /tmp/tmp.V7o22rA34g /tmp/tmp.I5cr0cppLK ++ return 0 + FIRST_GCP_BACKUP=cron-scheduled-backup-gcp-cs-202511704157-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-gcp-cs-202511704157-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0e423bseBl +++ mktemp ++ local LAST_ERR=/tmp/tmp.jHYtcGOdY0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0e423bseBl ++ cat /tmp/tmp.jHYtcGOdY0 ++ rm /tmp/tmp.0e423bseBl /tmp/tmp.jHYtcGOdY0 ++ return 0 + JOB_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-202511704157-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-gcp-cs-202511704157 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XKw3RmcM5q +++ mktemp ++ local LAST_ERR=/tmp/tmp.xmfWDAUuCP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XKw3RmcM5q ++ cat /tmp/tmp.xmfWDAUuCP ++ rm /tmp/tmp.XKw3RmcM5q /tmp/tmp.xmfWDAUuCP ++ return 0 + POD_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-202511704157-q6fav-kh6wk ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ grep azure ++ head -n1 +++ mktemp ++ cut -d: -f1 ++ local LAST_OUT=/tmp/tmp.HDvoYQXiiI +++ mktemp ++ local LAST_ERR=/tmp/tmp.6NJCvWvpaE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HDvoYQXiiI ++ cat /tmp/tmp.6NJCvWvpaE ++ rm /tmp/tmp.HDvoYQXiiI /tmp/tmp.6NJCvWvpaE ++ return 0 + FIRST_AZURE_BACKUP=cron-scheduled-backup-azure-blob-202511704339-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-azure-blob-202511704339-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.volUvT0FPY +++ mktemp ++ local LAST_ERR=/tmp/tmp.2NghKivUWR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.volUvT0FPY ++ cat /tmp/tmp.2NghKivUWR ++ rm /tmp/tmp.volUvT0FPY /tmp/tmp.2NghKivUWR ++ return 0 + JOB_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-202511704339-q6fav ++ kubectl_bin get pods ++ awk '{print $1}' ++ grep xb-cron-scheduled-backup-azure-blob-202511704339 +++ mktemp ++ local LAST_OUT=/tmp/tmp.oj77qccN7t +++ mktemp ++ local LAST_ERR=/tmp/tmp.sgGbnQUnBW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oj77qccN7t ++ cat /tmp/tmp.sgGbnQUnBW ++ rm /tmp/tmp.oj77qccN7t /tmp/tmp.sgGbnQUnBW ++ return 0 + POD_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-202511704339-q6fav-c5g58 ++ kubectl_bin get pxc-backup '' -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ cut -c 6- +++ mktemp ++ local LAST_OUT=/tmp/tmp.6AsO7JC3co +++ mktemp ++ local LAST_ERR=/tmp/tmp.cyGrSGat8J ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '' -o 'jsonpath={.status.destination}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 ']' ++ '[' 1 == 1 ']' ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '' -o 'jsonpath={.status.destination}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 ']' ++ '[' 1 == 1 ']' ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '' -o 'jsonpath={.status.destination}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 ']' ++ '[' 1 == 1 ']' ++ sleep 0 ++ cat /tmp/tmp.6AsO7JC3co ++ cat /tmp/tmp.cyGrSGat8J error: resource name may not be empty ++ rm /tmp/tmp.6AsO7JC3co /tmp/tmp.cyGrSGat8J ++ return 1 + BACKUP_DEST_AWS= ++ kubectl_bin get pxc-backup cron-scheduled-backup-gcp-cs-202511704157-q6fav -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ cut -c 6- +++ mktemp ++ local LAST_OUT=/tmp/tmp.XiG4dZSOA2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.AdRqhOJh0M ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-gcp-cs-202511704157-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XiG4dZSOA2 ++ cat /tmp/tmp.AdRqhOJh0M ++ rm /tmp/tmp.XiG4dZSOA2 /tmp/tmp.AdRqhOJh0M ++ return 0 + BACKUP_DEST_GCP=operator-testing/scheduled-backup-2025-11-07-00:41:57-full ++ kubectl_bin get pxc-backup cron-scheduled-backup-azure-blob-202511704339-q6fav -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ cut -c 9- +++ mktemp ++ local LAST_OUT=/tmp/tmp.AhPKgPOc9b +++ mktemp ++ local LAST_ERR=/tmp/tmp.QlGikwquoR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-azure-blob-202511704339-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AhPKgPOc9b ++ cat /tmp/tmp.QlGikwquoR ++ rm /tmp/tmp.AhPKgPOc9b /tmp/tmp.QlGikwquoR ++ return 0 + BACKUP_DEST_AZURE=operator-testing/scheduled-backup-2025-11-07-00:43:39-full + desc 'Check backup existence' + set +o xtrace ----------------------------------------------------------------------------------- Check backup existence ----------------------------------------------------------------------------------- + check_backup_existence https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 aws-s3 + path=https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + storage_name=aws-s3 + retry=0 ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 0 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 1 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 2 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 3 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 4 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 5 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 6 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 7 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 8 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 9 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 10 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 11 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 12 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 13 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 14 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 15 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 16 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 17 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 18 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 19 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 20 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 21 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 22 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 23 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 24 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 25 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 26 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 27 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 28 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 29 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 30 -ge 30 ']' + echo 'max retry count 30 reached. something went wrong with operator or kubernetes cluster' max retry count 30 reached. something went wrong with operator or kubernetes cluster + echo 'Backup was not found in bucket -- aws-s3' Backup was not found in bucket -- aws-s3 + exit 1