Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/logs/scheduled-backup-8-0.log Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 + main + create_infra scheduled-backup-32093 + local ns=scheduled-backup-32093 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n scheduled-backup-21390 scheduled-backup --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.cU5ZytWlbp ++ mktemp + local LAST_ERR=/tmp/tmp.Cq3ik5wQrZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cU5ZytWlbp perconaxtradbcluster.pxc.percona.com "scheduled-backup" deleted from scheduled-backup-21390 namespace + cat /tmp/tmp.Cq3ik5wQrZ + rm /tmp/tmp.cU5ZytWlbp /tmp/tmp.Cq3ik5wQrZ + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.khJC8lZ0Gp ++ mktemp + local LAST_ERR=/tmp/tmp.SeKliZhoxc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.khJC8lZ0Gp perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-aws-s3-2025124123713-q6fav" deleted from scheduled-backup-21390 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-pvc-2025124123453-q6fav" deleted from scheduled-backup-21390 namespace + cat /tmp/tmp.SeKliZhoxc + rm /tmp/tmp.khJC8lZ0Gp /tmp/tmp.SeKliZhoxc + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.IzNUlsGPYf ++ mktemp + local LAST_ERR=/tmp/tmp.E33VflexkY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IzNUlsGPYf No resources found + cat /tmp/tmp.E33VflexkY + rm /tmp/tmp.IzNUlsGPYf /tmp/tmp.E33VflexkY + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ++ mktemp ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.N0ZebaM7mY + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + local LAST_OUT=/tmp/tmp.uAninKfxUt ++ mktemp + local LAST_ERR=/tmp/tmp.JHNdeAwnQq + local exit_status=0 ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.kMnztg1SCf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + xargs kubectl delete ns + awk '{print$1}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.N0ZebaM7mY + cat /tmp/tmp.JHNdeAwnQq + rm /tmp/tmp.N0ZebaM7mY /tmp/tmp.JHNdeAwnQq + return 0 namespace "scheduled-backup-21390" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uAninKfxUt namespace "pxc-operator" deleted + cat /tmp/tmp.kMnztg1SCf + rm /tmp/tmp.uAninKfxUt /tmp/tmp.kMnztg1SCf + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.3JYt8e93N5 ++ mktemp + local LAST_ERR=/tmp/tmp.5p7zOpEeiX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3JYt8e93N5 namespace/pxc-operator created + cat /tmp/tmp.5p7zOpEeiX + rm /tmp/tmp.3JYt8e93N5 /tmp/tmp.5p7zOpEeiX + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZvoXyczb9X +++ mktemp ++ local LAST_ERR=/tmp/tmp.IA5SFy9mcc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZvoXyczb9X ++ cat /tmp/tmp.IA5SFy9mcc ++ rm /tmp/tmp.ZvoXyczb9X /tmp/tmp.IA5SFy9mcc ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster8 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.KLmDZxLP3o ++ mktemp + local LAST_ERR=/tmp/tmp.DUZpPDP3io + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster8 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KLmDZxLP3o Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster8" modified. + cat /tmp/tmp.DUZpPDP3io + rm /tmp/tmp.KLmDZxLP3o /tmp/tmp.DUZpPDP3io + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.KVvadOBC9I ++ mktemp + local LAST_ERR=/tmp/tmp.eZlmNsiqLO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KVvadOBC9I customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.eZlmNsiqLO + rm /tmp/tmp.KVvadOBC9I /tmp/tmp.eZlmNsiqLO + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/deploy/cw-rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.stucV70Any ++ mktemp + local LAST_ERR=/tmp/tmp.8e5uizp75V + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.stucV70Any clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.8e5uizp75V + rm /tmp/tmp.stucV70Any /tmp/tmp.8e5uizp75V + return 0 + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/deploy/cw-operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.CMpiGCxhF4 ++ mktemp + local LAST_ERR=/tmp/tmp.8DHlpkk3Bf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CMpiGCxhF4 deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.8DHlpkk3Bf + rm /tmp/tmp.CMpiGCxhF4 /tmp/tmp.8DHlpkk3Bf + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.0aqDr3Avx7 ++ mktemp + local LAST_ERR=/tmp/tmp.57LD1tRYXp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0aqDr3Avx7 pod/percona-xtradb-cluster-operator-84658c6c47-sz2f5 condition met + cat /tmp/tmp.57LD1tRYXp + rm /tmp/tmp.0aqDr3Avx7 /tmp/tmp.57LD1tRYXp + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.DCPbULSIwb +++ mktemp ++ local LAST_ERR=/tmp/tmp.g8Ld8WoMia ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DCPbULSIwb ++ cat /tmp/tmp.g8Ld8WoMia ++ rm /tmp/tmp.DCPbULSIwb /tmp/tmp.g8Ld8WoMia ++ return 0 + wait_pod percona-xtradb-cluster-operator-84658c6c47-sz2f5 480 pxc-operator + local pod=percona-xtradb-cluster-operator-84658c6c47-sz2f5 + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-84658c6c47-sz2f5 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-84658c6c47-sz2f5 condition met waiting for pod/percona-xtradb-cluster-operator-84658c6c47-sz2f5 to become Ready.Ok + sleep 3 + create_namespace scheduled-backup-32093 + local namespace=scheduled-backup-32093 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces scheduled-backup-32093' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces scheduled-backup-32093 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace scheduled-backup-32093 ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.HnpIlvzPwJ ++ mktemp + local LAST_OUT=/tmp/tmp.caE4Jk61ha ++ mktemp + local LAST_ERR=/tmp/tmp.ywVBhLxeHB + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.Q4osHseBsb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-32093 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-32093 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HnpIlvzPwJ + cat /tmp/tmp.ywVBhLxeHB + rm /tmp/tmp.HnpIlvzPwJ /tmp/tmp.ywVBhLxeHB + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-32093 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.caE4Jk61ha + cat /tmp/tmp.Q4osHseBsb Error from server (NotFound): namespaces "scheduled-backup-32093" not found + rm /tmp/tmp.caE4Jk61ha /tmp/tmp.Q4osHseBsb + return 1 + : + wait_for_delete namespace/scheduled-backup-32093 + local res=namespace/scheduled-backup-32093 + echo -n 'waiting for namespace/scheduled-backup-32093 to be deleted' waiting for namespace/scheduled-backup-32093 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "scheduled-backup-32093" not found + desc 'create namespace scheduled-backup-32093' + set +o xtrace ----------------------------------------------------------------------------------- create namespace scheduled-backup-32093 ----------------------------------------------------------------------------------- + kubectl_bin create namespace scheduled-backup-32093 ++ mktemp + local LAST_OUT=/tmp/tmp.OgPwKUWPtA ++ mktemp + local LAST_ERR=/tmp/tmp.tla22mpj1l + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace scheduled-backup-32093 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OgPwKUWPtA namespace/scheduled-backup-32093 created + cat /tmp/tmp.tla22mpj1l + rm /tmp/tmp.OgPwKUWPtA /tmp/tmp.tla22mpj1l + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.xIcHb0F7Px +++ mktemp ++ local LAST_ERR=/tmp/tmp.92H7K8aZD3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xIcHb0F7Px ++ cat /tmp/tmp.92H7K8aZD3 ++ rm /tmp/tmp.xIcHb0F7Px /tmp/tmp.92H7K8aZD3 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster8 --namespace=scheduled-backup-32093 ++ mktemp + local LAST_OUT=/tmp/tmp.GplrcsSL3E ++ mktemp + local LAST_ERR=/tmp/tmp.DLCY6IO2fW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster8 --namespace=scheduled-backup-32093 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GplrcsSL3E Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster8" modified. + cat /tmp/tmp.DLCY6IO2fW + rm /tmp/tmp.GplrcsSL3E /tmp/tmp.DLCY6IO2fW + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.lHZCeWw2WF ++ mktemp + local LAST_ERR=/tmp/tmp.w4O5Eqgq9w + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lHZCeWw2WF secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.w4O5Eqgq9w + rm /tmp/tmp.lHZCeWw2WF /tmp/tmp.w4O5Eqgq9w + return 0 + start_minio + deploy_helm scheduled-backup-32093 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "hashicorp" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Thu Dec 4 12:45:14 2025 NAMESPACE: scheduled-backup-32093 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.scheduled-backup-32093.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-32093 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-32093 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-32093 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-32093 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vxMWu6IRLQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.5vrQfsAphp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vxMWu6IRLQ ++ cat /tmp/tmp.5vrQfsAphp ++ rm /tmp/tmp.vxMWu6IRLQ /tmp/tmp.5vrQfsAphp ++ return 0 + MINIO_POD=minio-service-55fcc5d75f-26x7v + wait_pod minio-service-55fcc5d75f-26x7v + local pod=minio-service-55fcc5d75f-26x7v + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo minio-service-55fcc5d75f-26x7v ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/minio-service-55fcc5d75f-26x7v condition met waiting for pod/minio-service-55fcc5d75f-26x7v to become Ready.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.JFcZtB4GCz ++ mktemp + local LAST_ERR=/tmp/tmp.lkzCGanb8y + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JFcZtB4GCz make_bucket: operator-testing pod "aws-cli" deleted from scheduled-backup-32093 namespace + cat /tmp/tmp.lkzCGanb8y + rm /tmp/tmp.JFcZtB4GCz /tmp/tmp.lkzCGanb8y + return 0 + cluster=scheduled-backup + cat - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.1czRMEtJPz ++ mktemp + local LAST_ERR=/tmp/tmp.m3Sov0b7XC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1czRMEtJPz priorityclass.scheduling.k8s.io/high-priority configured + cat /tmp/tmp.m3Sov0b7XC + rm /tmp/tmp.1czRMEtJPz /tmp/tmp.m3Sov0b7XC + return 0 + spinup_pxc scheduled-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local cluster=scheduled-backup + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.AygaOC89O5 ++ mktemp + local LAST_ERR=/tmp/tmp.G7YPaMKgmA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AygaOC89O5 secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.G7YPaMKgmA + rm /tmp/tmp.AygaOC89O5 /tmp/tmp.G7YPaMKgmA + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-32093~ + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.IcoND2hNqG ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_ERR=/tmp/tmp.86QUj5jw3I + local exit_status=0 + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IcoND2hNqG deployment.apps/pxc-client created + cat /tmp/tmp.86QUj5jw3I + rm /tmp/tmp.IcoND2hNqG /tmp/tmp.86QUj5jw3I + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml '' + kubectl_bin apply -f - + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-32093~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.FlxqFNd2E1 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + local LAST_ERR=/tmp/tmp.f4wKXZYNsA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FlxqFNd2E1 perconaxtradbcluster.pxc.percona.com/scheduled-backup created + cat /tmp/tmp.f4wKXZYNsA + rm /tmp/tmp.FlxqFNd2E1 /tmp/tmp.f4wKXZYNsA + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy scheduled-backup ++ local target_cluster=scheduled-backup +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qcsRtSExar ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gGC3nAPSv4 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.qcsRtSExar +++ cat /tmp/tmp.gGC3nAPSv4 +++ rm /tmp/tmp.qcsRtSExar /tmp/tmp.gGC3nAPSv4 +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dPDs7x4o3L ++++ mktemp +++ local LAST_ERR=/tmp/tmp.n1r5cAcHa1 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.dPDs7x4o3L +++ cat /tmp/tmp.n1r5cAcHa1 +++ rm /tmp/tmp.dPDs7x4o3L /tmp/tmp.n1r5cAcHa1 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scheduled-backup-proxysql ++ return + local proxy=scheduled-backup-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-32093 ++ mktemp + local LAST_OUT=/tmp/tmp.AXxSI12DrK ++ mktemp + local LAST_ERR=/tmp/tmp.L14YUU0FQc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-32093 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-32093 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-32093 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.AXxSI12DrK + cat /tmp/tmp.L14YUU0FQc error: no matching resources found + rm /tmp/tmp.AXxSI12DrK /tmp/tmp.L14YUU0FQc + return 1 + true + wait_for_running scheduled-backup-proxysql 1 + local name=scheduled-backup-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-proxysql-0 480 + local pod=scheduled-backup-proxysql-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo scheduled-backup-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace pod/scheduled-backup-proxysql-0 condition met waiting for pod/scheduled-backup-proxysql-0 to become Ready.Ok + wait_for_running scheduled-backup-pxc 3 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 480 + local pod=scheduled-backup-pxc-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 480 + local pod=scheduled-backup-pxc-1 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-1 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 480 + local pod=scheduled-backup-pxc-2 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ base64 --decode ++ local LAST_OUT=/tmp/tmp.C2HzmfYcJ6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.bQ3nYlC1uc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.C2HzmfYcJ6 ++ cat /tmp/tmp.bQ3nYlC1uc ++ rm /tmp/tmp.C2HzmfYcJ6 /tmp/tmp.bQ3nYlC1uc ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DhlmsqND5X +++ mktemp ++ local LAST_ERR=/tmp/tmp.5vPuVDvSC7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DhlmsqND5X ++ cat /tmp/tmp.5vPuVDvSC7 ++ rm /tmp/tmp.DhlmsqND5X /tmp/tmp.5vPuVDvSC7 ++ return 0 + client_pod=pxc-client-59944c5bbf-9gbxk + wait_pod pxc-client-59944c5bbf-9gbxk + local pod=pxc-client-59944c5bbf-9gbxk + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-9gbxk ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-9gbxk condition met waiting for pod/pxc-client-59944c5bbf-9gbxk to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.APD8kgZbDF +++ mktemp ++ local LAST_ERR=/tmp/tmp.JIILYLpzlc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.APD8kgZbDF ++ cat /tmp/tmp.JIILYLpzlc ++ rm /tmp/tmp.APD8kgZbDF /tmp/tmp.JIILYLpzlc ++ return 0 + client_pod=pxc-client-59944c5bbf-9gbxk + wait_pod pxc-client-59944c5bbf-9gbxk + local pod=pxc-client-59944c5bbf-9gbxk + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-59944c5bbf-9gbxk + local container= + set +o xtrace pod/pxc-client-59944c5bbf-9gbxk condition met waiting for pod/pxc-client-59944c5bbf-9gbxk to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q6GYwkKt2G +++ mktemp ++ local LAST_ERR=/tmp/tmp.I4ea9UynaO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.q6GYwkKt2G ++ cat /tmp/tmp.I4ea9UynaO ++ rm /tmp/tmp.q6GYwkKt2G /tmp/tmp.I4ea9UynaO ++ return 0 + client_pod=pxc-client-59944c5bbf-9gbxk + wait_pod pxc-client-59944c5bbf-9gbxk + local pod=pxc-client-59944c5bbf-9gbxk + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-59944c5bbf-9gbxk + local container= + set +o xtrace pod/pxc-client-59944c5bbf-9gbxk condition met waiting for pod/pxc-client-59944c5bbf-9gbxk to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.wsRFTpVhIh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.wsRFTpVhIh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.wsRFTpVhIh/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uDXnlN5Auy +++ mktemp ++ local LAST_ERR=/tmp/tmp.5cr6sQHzsI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uDXnlN5Auy ++ cat /tmp/tmp.5cr6sQHzsI ++ rm /tmp/tmp.uDXnlN5Auy /tmp/tmp.5cr6sQHzsI ++ return 0 + client_pod=pxc-client-59944c5bbf-9gbxk + wait_pod pxc-client-59944c5bbf-9gbxk + local pod=pxc-client-59944c5bbf-9gbxk + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-9gbxk ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-9gbxk condition met waiting for pod/pxc-client-59944c5bbf-9gbxk to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.wsRFTpVhIh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.wsRFTpVhIh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.wsRFTpVhIh/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YusNniALCN +++ mktemp ++ local LAST_ERR=/tmp/tmp.3eOncG7U5u ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YusNniALCN ++ cat /tmp/tmp.3eOncG7U5u ++ rm /tmp/tmp.YusNniALCN /tmp/tmp.3eOncG7U5u ++ return 0 + client_pod=pxc-client-59944c5bbf-9gbxk + wait_pod pxc-client-59944c5bbf-9gbxk + local pod=pxc-client-59944c5bbf-9gbxk + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-59944c5bbf-9gbxk + local container= + set +o xtrace pod/pxc-client-59944c5bbf-9gbxk condition met waiting for pod/pxc-client-59944c5bbf-9gbxk to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.wsRFTpVhIh/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.wsRFTpVhIh/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.wsRFTpVhIh/select-1.sql + is_keyring_plugin_in_use scheduled-backup + local cluster=scheduled-backup + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + grep -E -o 'early-plugin-load=keyring_\w+.so' ++ mktemp + local LAST_OUT=/tmp/tmp.U9rLQbHz98 ++ mktemp + local LAST_ERR=/tmp/tmp.jdw69QaX2I + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.U9rLQbHz98 + cat /tmp/tmp.jdw69QaX2I Unable to use a TTY - input is not a terminal or the right kind of file + rm /tmp/tmp.U9rLQbHz98 /tmp/tmp.jdw69QaX2I + return 0 + return 1 + sleep 20 + desc 'add backups schedule for pvc storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for pvc storage ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.bjzEVmwi3e +++ mktemp ++ local LAST_ERR=/tmp/tmp.C8R9ei8zNv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bjzEVmwi3e ++ cat /tmp/tmp.C8R9ei8zNv ++ rm /tmp/tmp.bjzEVmwi3e /tmp/tmp.C8R9ei8zNv ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster8 --namespace=scheduled-backup-32093 ++ mktemp + local LAST_OUT=/tmp/tmp.sojxB2q0Vm ++ mktemp + local LAST_ERR=/tmp/tmp.dwdEalotoF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster8 --namespace=scheduled-backup-32093 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sojxB2q0Vm Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster8" modified. + cat /tmp/tmp.dwdEalotoF + rm /tmp/tmp.sojxB2q0Vm /tmp/tmp.dwdEalotoF + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local pvc_name= + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_OUT=/tmp/tmp.9ZOGzlmO8c + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-32093~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + local LAST_ERR=/tmp/tmp.ZsSV7KPyLG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9ZOGzlmO8c perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.ZsSV7KPyLG + rm /tmp/tmp.9ZOGzlmO8c /tmp/tmp.ZsSV7KPyLG + return 0 + label_node ++ kubectl_bin get nodes --no-headers=true ++ grep -v master ++ awk '{print $1}' +++ mktemp ++ head -n1 ++ local LAST_OUT=/tmp/tmp.1hFjWeBi4C +++ mktemp ++ local LAST_ERR=/tmp/tmp.xui1SUhcdi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get nodes --no-headers=true ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1hFjWeBi4C ++ cat /tmp/tmp.xui1SUhcdi ++ rm /tmp/tmp.1hFjWeBi4C /tmp/tmp.xui1SUhcdi ++ return 0 + LABELED_NODE=gke-jen-pxc-2076-ee1be0b-default-pool-3202209d-1ntt + kubectl_bin label nodes gke-jen-pxc-2076-ee1be0b-default-pool-3202209d-1ntt backupWorker=True --overwrite ++ mktemp + local LAST_OUT=/tmp/tmp.2FP4oa95U4 ++ mktemp + local LAST_ERR=/tmp/tmp.WX4YfWmTUM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label nodes gke-jen-pxc-2076-ee1be0b-default-pool-3202209d-1ntt backupWorker=True --overwrite + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2FP4oa95U4 node/gke-jen-pxc-2076-ee1be0b-default-pool-3202209d-1ntt not labeled + cat /tmp/tmp.WX4YfWmTUM + rm /tmp/tmp.2FP4oa95U4 /tmp/tmp.WX4YfWmTUM + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-32093~ + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.kV9RyqLKJi + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_ERR=/tmp/tmp.h5GRgWIfB6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kV9RyqLKJi perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.h5GRgWIfB6 + rm /tmp/tmp.kV9RyqLKJi /tmp/tmp.h5GRgWIfB6 + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.dDHOtrYF7j +++ mktemp ++ local LAST_ERR=/tmp/tmp.NtJeyuTGjX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dDHOtrYF7j ++ cat /tmp/tmp.NtJeyuTGjX ++ rm /tmp/tmp.dDHOtrYF7j /tmp/tmp.NtJeyuTGjX ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.B0FT9BDMUF +++ mktemp ++ local LAST_ERR=/tmp/tmp.EcpLRVjKTE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.B0FT9BDMUF ++ cat /tmp/tmp.EcpLRVjKTE ++ rm /tmp/tmp.B0FT9BDMUF /tmp/tmp.EcpLRVjKTE ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-0 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.KHeQDajYpv ++ mktemp + local LAST_ERR=/tmp/tmp.MbJ2OcmYXu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KHeQDajYpv NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-202512412534-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202512412534-q6fav-20251204125304-522b555d Running 21s + cat /tmp/tmp.MbJ2OcmYXu + rm /tmp/tmp.KHeQDajYpv /tmp/tmp.MbJ2OcmYXu + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.XLH3cdSoDS ++ mktemp + local LAST_ERR=/tmp/tmp.bCLwAKBg9e + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XLH3cdSoDS NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-26x7v 1/1 Running 0 8m9s pxc-client-59944c5bbf-9gbxk 2/2 Running 0 6m56s scheduled-backup-proxysql-0 3/3 Running 0 6m52s scheduled-backup-proxysql-1 3/3 Running 0 6m34s scheduled-backup-pxc-0 1/1 Running 0 6m52s scheduled-backup-pxc-1 1/1 Running 0 5m39s scheduled-backup-pxc-2 1/1 Running 0 4m28s xb-cron-scheduled-backup-pvc-202512412534-q6fav-n8vt7 1/1 Running 0 22s + cat /tmp/tmp.bCLwAKBg9e + rm /tmp/tmp.XLH3cdSoDS /tmp/tmp.bCLwAKBg9e + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.bm9ukp38mb +++ mktemp ++ local LAST_ERR=/tmp/tmp.pHO5J3x4bR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bm9ukp38mb ++ cat /tmp/tmp.pHO5J3x4bR ++ rm /tmp/tmp.bm9ukp38mb /tmp/tmp.pHO5J3x4bR ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.Vv2dMCSp4h +++ mktemp ++ local LAST_ERR=/tmp/tmp.eoaZaWb7bt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Vv2dMCSp4h ++ cat /tmp/tmp.eoaZaWb7bt ++ rm /tmp/tmp.Vv2dMCSp4h /tmp/tmp.eoaZaWb7bt ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local pvc_name= ++ mktemp + local LAST_OUT=/tmp/tmp.fFe9qN0AqN + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local LAST_ERR=/tmp/tmp.x4uk7AyiJH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-32093~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fFe9qN0AqN perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.x4uk7AyiJH + rm /tmp/tmp.fFe9qN0AqN /tmp/tmp.x4uk7AyiJH + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba#' + local LAST_OUT=/tmp/tmp.FJxMo350aG + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-32093~ + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + local LAST_ERR=/tmp/tmp.BlFUSXZLGu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FJxMo350aG perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.BlFUSXZLGu + rm /tmp/tmp.FJxMo350aG /tmp/tmp.BlFUSXZLGu + return 0 + wait_all_backups ++ get_running_backups_amount ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.6Y7eJvXDzX +++ mktemp ++ local LAST_ERR=/tmp/tmp.DyMhUiYxd8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6Y7eJvXDzX ++ cat /tmp/tmp.DyMhUiYxd8 ++ rm /tmp/tmp.6Y7eJvXDzX /tmp/tmp.DyMhUiYxd8 ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nOOwKzJs5K ++ grep :Failed +++ mktemp ++ local LAST_ERR=/tmp/tmp.XLhv5k751E ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nOOwKzJs5K ++ cat /tmp/tmp.XLhv5k751E ++ rm /tmp/tmp.nOOwKzJs5K /tmp/tmp.XLhv5k751E ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.HU4orK7JMm ++ mktemp + local LAST_ERR=/tmp/tmp.2PLYOFgfim + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HU4orK7JMm NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125451-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:54:51-full Running 25s cron-scheduled-backup-pvc-202512412534-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202512412534-q6fav-20251204125304-522b555d Succeeded 93s 2m12s + cat /tmp/tmp.2PLYOFgfim + rm /tmp/tmp.HU4orK7JMm /tmp/tmp.2PLYOFgfim + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.4rM9fydPPT ++ mktemp + local LAST_ERR=/tmp/tmp.v7wUieEOmX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4rM9fydPPT NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-26x7v 1/1 Running 0 10m pxc-client-59944c5bbf-9gbxk 2/2 Running 0 8m47s scheduled-backup-proxysql-0 3/3 Running 0 8m43s scheduled-backup-proxysql-1 3/3 Running 0 8m25s scheduled-backup-pxc-0 1/1 Running 0 8m43s scheduled-backup-pxc-1 1/1 Running 0 7m30s scheduled-backup-pxc-2 1/1 Running 0 6m19s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-pgn4x 1/1 Running 0 26s xb-cron-scheduled-backup-pvc-202512412534-q6fav-n8vt7 0/1 Completed 0 2m13s + cat /tmp/tmp.v7wUieEOmX + rm /tmp/tmp.4rM9fydPPT /tmp/tmp.v7wUieEOmX + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.zDHWGGC9GQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.13b7J6yxNl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zDHWGGC9GQ ++ cat /tmp/tmp.13b7J6yxNl ++ rm /tmp/tmp.zDHWGGC9GQ /tmp/tmp.13b7J6yxNl ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZtQSD9qiqh +++ mktemp ++ local LAST_ERR=/tmp/tmp.rniKXhzlPV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZtQSD9qiqh ++ cat /tmp/tmp.rniKXhzlPV ++ rm /tmp/tmp.ZtQSD9qiqh /tmp/tmp.rniKXhzlPV ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.lOMhWaKKpt ++ mktemp + local LAST_ERR=/tmp/tmp.SSViw9PHwR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lOMhWaKKpt NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125451-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:54:51-full Running 64s cron-scheduled-backup-pvc-202512412534-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202512412534-q6fav-20251204125304-522b555d Succeeded 2m12s 2m51s + cat /tmp/tmp.SSViw9PHwR + rm /tmp/tmp.lOMhWaKKpt /tmp/tmp.SSViw9PHwR + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.OQaQ98Jr68 ++ mktemp + local LAST_ERR=/tmp/tmp.Qu4ndeUo3p + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OQaQ98Jr68 NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-26x7v 1/1 Running 0 10m pxc-client-59944c5bbf-9gbxk 2/2 Running 0 9m25s scheduled-backup-proxysql-0 3/3 Running 0 9m21s scheduled-backup-proxysql-1 3/3 Running 0 9m3s scheduled-backup-pxc-0 1/1 Running 0 9m21s scheduled-backup-pxc-1 1/1 Running 0 8m8s scheduled-backup-pxc-2 1/1 Running 0 6m57s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-klwvx 1/1 Running 0 24s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-pgn4x 0/1 Error 0 64s xb-cron-scheduled-backup-pvc-202512412534-q6fav-n8vt7 0/1 Completed 0 2m51s + cat /tmp/tmp.Qu4ndeUo3p + rm /tmp/tmp.OQaQ98Jr68 /tmp/tmp.Qu4ndeUo3p + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.neeGOZcs58 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Feoz6idtPK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.neeGOZcs58 ++ cat /tmp/tmp.Feoz6idtPK ++ rm /tmp/tmp.neeGOZcs58 /tmp/tmp.Feoz6idtPK ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.TGN2q9cEwO +++ mktemp ++ local LAST_ERR=/tmp/tmp.ut2dPf9IB4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TGN2q9cEwO ++ cat /tmp/tmp.ut2dPf9IB4 ++ rm /tmp/tmp.TGN2q9cEwO /tmp/tmp.ut2dPf9IB4 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.tBpM2iBsv6 ++ mktemp + local LAST_ERR=/tmp/tmp.YWb4g3PJHe + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tBpM2iBsv6 NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125451-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:54:51-full Running 102s cron-scheduled-backup-pvc-202512412534-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202512412534-q6fav-20251204125304-522b555d Succeeded 2m50s 3m29s + cat /tmp/tmp.YWb4g3PJHe + rm /tmp/tmp.tBpM2iBsv6 /tmp/tmp.YWb4g3PJHe + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.6EVZ04DVZY ++ mktemp + local LAST_ERR=/tmp/tmp.A0wuOkU2aE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6EVZ04DVZY NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-26x7v 1/1 Running 0 11m pxc-client-59944c5bbf-9gbxk 2/2 Running 0 10m scheduled-backup-proxysql-0 3/3 Running 0 10m scheduled-backup-proxysql-1 3/3 Running 0 9m42s scheduled-backup-pxc-0 1/1 Running 0 10m scheduled-backup-pxc-1 1/1 Running 0 8m47s scheduled-backup-pxc-2 1/1 Running 0 7m36s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-klwvx 1/1 Running 0 63s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-pgn4x 0/1 Error 0 103s xb-cron-scheduled-backup-pvc-202512412534-q6fav-n8vt7 0/1 Completed 0 3m30s + cat /tmp/tmp.A0wuOkU2aE + rm /tmp/tmp.6EVZ04DVZY /tmp/tmp.A0wuOkU2aE + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.FsZXVOEQMh +++ mktemp ++ local LAST_ERR=/tmp/tmp.Wod7mNi99W ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FsZXVOEQMh ++ cat /tmp/tmp.Wod7mNi99W ++ rm /tmp/tmp.FsZXVOEQMh /tmp/tmp.Wod7mNi99W ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.vYsqIOSD7L +++ mktemp ++ local LAST_ERR=/tmp/tmp.LYP8AvBLyK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vYsqIOSD7L ++ cat /tmp/tmp.LYP8AvBLyK ++ rm /tmp/tmp.vYsqIOSD7L /tmp/tmp.LYP8AvBLyK ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.RcCPVVuojG ++ mktemp + local LAST_ERR=/tmp/tmp.vizw1aTNHm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RcCPVVuojG NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125451-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:54:51-full Running 2m23s cron-scheduled-backup-pvc-202512412534-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202512412534-q6fav-20251204125304-522b555d Succeeded 3m31s 4m10s + cat /tmp/tmp.vizw1aTNHm + rm /tmp/tmp.RcCPVVuojG /tmp/tmp.vizw1aTNHm + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.bV9Y8IO7sn ++ mktemp + local LAST_ERR=/tmp/tmp.a5SMJaLXOr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bV9Y8IO7sn NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-26x7v 1/1 Running 0 11m pxc-client-59944c5bbf-9gbxk 2/2 Running 0 10m scheduled-backup-proxysql-0 3/3 Running 0 10m scheduled-backup-proxysql-1 3/3 Running 0 10m scheduled-backup-pxc-0 1/1 Running 0 10m scheduled-backup-pxc-1 1/1 Running 0 9m28s scheduled-backup-pxc-2 1/1 Running 0 8m17s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-klwvx 0/1 Error 0 104s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-pgn4x 0/1 Error 0 2m24s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-q5jvp 1/1 Running 0 7s xb-cron-scheduled-backup-pvc-202512412534-q6fav-n8vt7 0/1 Completed 0 4m11s + cat /tmp/tmp.a5SMJaLXOr + rm /tmp/tmp.bV9Y8IO7sn /tmp/tmp.a5SMJaLXOr + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.rAUBeBgOCt +++ mktemp ++ local LAST_ERR=/tmp/tmp.mlkcYNWuCD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rAUBeBgOCt ++ cat /tmp/tmp.mlkcYNWuCD ++ rm /tmp/tmp.rAUBeBgOCt /tmp/tmp.mlkcYNWuCD ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.TNjF6aHgmp ++ wc -l +++ mktemp ++ local LAST_ERR=/tmp/tmp.F7ae8JFnhQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TNjF6aHgmp ++ cat /tmp/tmp.F7ae8JFnhQ ++ rm /tmp/tmp.TNjF6aHgmp /tmp/tmp.F7ae8JFnhQ ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.v7OTm7ocUB ++ mktemp + local LAST_ERR=/tmp/tmp.Ki4IWCRTbf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.v7OTm7ocUB NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125451-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:54:51-full Running 3m5s cron-scheduled-backup-pvc-202512412534-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202512412534-q6fav-20251204125304-522b555d Succeeded 4m13s 4m52s + cat /tmp/tmp.Ki4IWCRTbf + rm /tmp/tmp.v7OTm7ocUB /tmp/tmp.Ki4IWCRTbf + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.Izm5fLYygM ++ mktemp + local LAST_ERR=/tmp/tmp.AL7eoXiRJw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Izm5fLYygM NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-26x7v 1/1 Running 0 12m pxc-client-59944c5bbf-9gbxk 2/2 Running 0 11m scheduled-backup-proxysql-0 3/3 Running 0 11m scheduled-backup-proxysql-1 3/3 Running 0 11m scheduled-backup-pxc-0 1/1 Running 0 11m scheduled-backup-pxc-1 1/1 Running 0 10m scheduled-backup-pxc-2 1/1 Running 0 8m59s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-klwvx 0/1 Error 0 2m26s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-pgn4x 0/1 Error 0 3m6s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-q5jvp 1/1 Running 0 49s xb-cron-scheduled-backup-pvc-202512412534-q6fav-n8vt7 0/1 Completed 0 4m53s + cat /tmp/tmp.AL7eoXiRJw + rm /tmp/tmp.Izm5fLYygM /tmp/tmp.AL7eoXiRJw + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.pL1adXZ1ek +++ mktemp ++ local LAST_ERR=/tmp/tmp.ChTNSzzFVP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pL1adXZ1ek ++ cat /tmp/tmp.ChTNSzzFVP ++ rm /tmp/tmp.pL1adXZ1ek /tmp/tmp.ChTNSzzFVP ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.eOcy1vzZfQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.6gIhG2cmzk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eOcy1vzZfQ ++ cat /tmp/tmp.6gIhG2cmzk ++ rm /tmp/tmp.eOcy1vzZfQ /tmp/tmp.6gIhG2cmzk ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-0 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.Xw6H36qPTg ++ mktemp + local LAST_ERR=/tmp/tmp.4eZBGxpnq7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Xw6H36qPTg NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125451-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:54:51-full Starting 3m45s cron-scheduled-backup-pvc-202512412534-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202512412534-q6fav-20251204125304-522b555d Succeeded 4m53s 5m32s + cat /tmp/tmp.4eZBGxpnq7 + rm /tmp/tmp.Xw6H36qPTg /tmp/tmp.4eZBGxpnq7 + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.omTWZZmSJd ++ mktemp + local LAST_ERR=/tmp/tmp.oXviGZb9kP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.omTWZZmSJd NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-26x7v 1/1 Running 0 13m pxc-client-59944c5bbf-9gbxk 2/2 Running 0 12m scheduled-backup-proxysql-0 3/3 Running 0 12m scheduled-backup-proxysql-1 3/3 Running 0 11m scheduled-backup-pxc-0 1/1 Running 0 12m scheduled-backup-pxc-1 1/1 Running 0 10m scheduled-backup-pxc-2 1/1 Running 0 9m38s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-klwvx 0/1 Error 0 3m5s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-pgn4x 0/1 Error 0 3m45s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-q5jvp 0/1 Error 0 88s xb-cron-scheduled-backup-pvc-202512412534-q6fav-n8vt7 0/1 Completed 0 5m32s + cat /tmp/tmp.oXviGZb9kP + rm /tmp/tmp.omTWZZmSJd /tmp/tmp.oXviGZb9kP + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.hwCyUoexq7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ATcb28f4YW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hwCyUoexq7 ++ cat /tmp/tmp.ATcb28f4YW ++ rm /tmp/tmp.hwCyUoexq7 /tmp/tmp.ATcb28f4YW ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.XkQ9J5wkyE +++ mktemp ++ local LAST_ERR=/tmp/tmp.B7uLNEjW5u ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XkQ9J5wkyE ++ cat /tmp/tmp.B7uLNEjW5u ++ rm /tmp/tmp.XkQ9J5wkyE /tmp/tmp.B7uLNEjW5u ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.PvfNPtJNzD ++ mktemp + local LAST_ERR=/tmp/tmp.OyGee5OZ5q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PvfNPtJNzD NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125451-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:54:51-full Starting 4m16s cron-scheduled-backup-pvc-202512412534-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202512412534-q6fav-20251204125304-522b555d Succeeded 5m24s 6m3s + cat /tmp/tmp.OyGee5OZ5q + rm /tmp/tmp.PvfNPtJNzD /tmp/tmp.OyGee5OZ5q + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.unU0EpCKG3 ++ mktemp + local LAST_ERR=/tmp/tmp.YEiZXcEOcp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.unU0EpCKG3 NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-26x7v 1/1 Running 0 13m pxc-client-59944c5bbf-9gbxk 2/2 Running 0 12m scheduled-backup-proxysql-0 3/3 Running 0 12m scheduled-backup-proxysql-1 3/3 Running 0 12m scheduled-backup-pxc-0 1/1 Running 0 12m scheduled-backup-pxc-1 1/1 Running 0 11m scheduled-backup-pxc-2 1/1 Running 0 10m xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-klwvx 0/1 Error 0 3m36s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-pgn4x 0/1 Error 0 4m16s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-q5jvp 0/1 Error 0 119s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-rfjg2 1/1 Running 0 2s xb-cron-scheduled-backup-pvc-202512412534-q6fav-n8vt7 0/1 Completed 0 6m3s + cat /tmp/tmp.YEiZXcEOcp + rm /tmp/tmp.unU0EpCKG3 /tmp/tmp.YEiZXcEOcp + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.5W87y5XJOI +++ mktemp ++ local LAST_ERR=/tmp/tmp.9t8LU3WDRR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5W87y5XJOI ++ cat /tmp/tmp.9t8LU3WDRR ++ rm /tmp/tmp.5W87y5XJOI /tmp/tmp.9t8LU3WDRR ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.B6raYtaHpn +++ mktemp ++ local LAST_ERR=/tmp/tmp.35AJmBCt9E ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.B6raYtaHpn ++ cat /tmp/tmp.35AJmBCt9E ++ rm /tmp/tmp.B6raYtaHpn /tmp/tmp.35AJmBCt9E ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-2 + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.gl33GjK8uZ ++ mktemp + local LAST_ERR=/tmp/tmp.4Wz0MJZK84 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gl33GjK8uZ NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125451-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:54:51-full Running 5m6s cron-scheduled-backup-pvc-202512412534-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202512412534-q6fav-20251204125304-522b555d Succeeded 6m14s 6m53s + cat /tmp/tmp.4Wz0MJZK84 + rm /tmp/tmp.gl33GjK8uZ /tmp/tmp.4Wz0MJZK84 + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.jGAdGATmwU ++ mktemp + local LAST_ERR=/tmp/tmp.BqzzCC4Y68 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jGAdGATmwU NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-26x7v 1/1 Running 0 14m pxc-client-59944c5bbf-9gbxk 2/2 Running 0 13m scheduled-backup-proxysql-0 3/3 Running 0 13m scheduled-backup-proxysql-1 3/3 Running 0 13m scheduled-backup-pxc-0 1/1 Running 0 13m scheduled-backup-pxc-1 1/1 Running 0 12m scheduled-backup-pxc-2 1/1 Running 0 11m xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-klwvx 0/1 Error 0 4m27s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-pgn4x 0/1 Error 0 5m7s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-q5jvp 0/1 Error 0 2m50s xb-cron-scheduled-backup-aws-s3-2025124125451-q6fav-rfjg2 1/1 Running 0 53s xb-cron-scheduled-backup-pvc-202512412534-q6fav-n8vt7 0/1 Completed 0 6m54s + cat /tmp/tmp.BqzzCC4Y68 + rm /tmp/tmp.jGAdGATmwU /tmp/tmp.BqzzCC4Y68 + return 0 + sleep 20 ++ get_running_backups_amount ++ grep -vE ':Succeeded|:Failed' ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xro6X0Gj9I +++ mktemp ++ local LAST_ERR=/tmp/tmp.5hVLTE6CF1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xro6X0Gj9I ++ cat /tmp/tmp.5hVLTE6CF1 ++ rm /tmp/tmp.xro6X0Gj9I /tmp/tmp.5hVLTE6CF1 ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.F92DOAA3Ya +++ mktemp ++ local LAST_ERR=/tmp/tmp.FD22lq2A2J ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.F92DOAA3Ya ++ cat /tmp/tmp.FD22lq2A2J ++ rm /tmp/tmp.F92DOAA3Ya /tmp/tmp.FD22lq2A2J ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-0 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-2 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.G4FTBIPuCH ++ mktemp + local LAST_ERR=/tmp/tmp.utU0hOcRl6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.G4FTBIPuCH NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125451-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:54:51-full Failed 5m45s cron-scheduled-backup-pvc-202512412534-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202512412534-q6fav-20251204125304-522b555d Succeeded 6m53s 7m32s + cat /tmp/tmp.utU0hOcRl6 + rm /tmp/tmp.G4FTBIPuCH /tmp/tmp.utU0hOcRl6 + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.1n7tvyZuRy ++ mktemp + local LAST_ERR=/tmp/tmp.4xbe8IxJsi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1n7tvyZuRy NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-26x7v 1/1 Running 0 15m pxc-client-59944c5bbf-9gbxk 2/2 Running 0 14m scheduled-backup-proxysql-0 3/3 Running 0 14m scheduled-backup-proxysql-1 3/3 Running 0 13m scheduled-backup-pxc-0 1/1 Running 0 14m scheduled-backup-pxc-1 1/1 Running 0 12m scheduled-backup-pxc-2 1/1 Running 0 11m xb-cron-scheduled-backup-pvc-202512412534-q6fav-n8vt7 0/1 Completed 0 7m33s + cat /tmp/tmp.4xbe8IxJsi + rm /tmp/tmp.1n7tvyZuRy /tmp/tmp.4xbe8IxJsi + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZQX69aMEQK +++ mktemp ++ local LAST_ERR=/tmp/tmp.yV8701qY4Q ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZQX69aMEQK ++ cat /tmp/tmp.yV8701qY4Q ++ rm /tmp/tmp.ZQX69aMEQK /tmp/tmp.yV8701qY4Q ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.hXIv479LuT +++ mktemp ++ local LAST_ERR=/tmp/tmp.gFcSfSkiHY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hXIv479LuT ++ cat /tmp/tmp.gFcSfSkiHY ++ rm /tmp/tmp.hXIv479LuT /tmp/tmp.gFcSfSkiHY ++ return 0 + [[ 1 -gt 0 ]] + echo 'One or more backups have been failed!\n' One or more backups have been failed!\n + desc 'LIST OF BACKUPS' + set +o xtrace ----------------------------------------------------------------------------------- LIST OF BACKUPS ----------------------------------------------------------------------------------- + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.PKOhKmStUa ++ mktemp + local LAST_ERR=/tmp/tmp.u1r4UIyfNT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PKOhKmStUa NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125451-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:54:51-full Failed 6m8s cron-scheduled-backup-pvc-202512412534-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202512412534-q6fav-20251204125304-522b555d Succeeded 7m16s 7m55s + cat /tmp/tmp.u1r4UIyfNT + rm /tmp/tmp.PKOhKmStUa /tmp/tmp.u1r4UIyfNT + return 0 + desc 'LIST OF PODS' + set +o xtrace ----------------------------------------------------------------------------------- LIST OF PODS ----------------------------------------------------------------------------------- + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.oQLJVOWkX3 ++ mktemp + local LAST_ERR=/tmp/tmp.vmjoPcf1bK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oQLJVOWkX3 NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-26x7v 1/1 Running 0 15m pxc-client-59944c5bbf-9gbxk 2/2 Running 0 14m scheduled-backup-proxysql-0 3/3 Running 0 14m scheduled-backup-proxysql-1 3/3 Running 0 14m scheduled-backup-pxc-0 1/1 Running 0 14m scheduled-backup-pxc-1 1/1 Running 0 13m scheduled-backup-pxc-2 1/1 Running 0 12m xb-cron-scheduled-backup-pvc-202512412534-q6fav-n8vt7 0/1 Completed 0 7m56s + cat /tmp/tmp.vmjoPcf1bK + rm /tmp/tmp.oQLJVOWkX3 /tmp/tmp.vmjoPcf1bK + return 0 + exit 1