Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/logs/scheduled-backup-5-7.log Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 + main + create_infra scheduled-backup-29415 + local ns=scheduled-backup-29415 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n scheduled-backup-32064 scheduled-backup --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.yJA6B41suf ++ mktemp + local LAST_ERR=/tmp/tmp.pOaylzjmEH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yJA6B41suf perconaxtradbcluster.pxc.percona.com "scheduled-backup" deleted from scheduled-backup-32064 namespace + cat /tmp/tmp.pOaylzjmEH + rm /tmp/tmp.yJA6B41suf /tmp/tmp.pOaylzjmEH + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.RVcPUWVz1N ++ mktemp + local LAST_ERR=/tmp/tmp.qTemdLkKFS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RVcPUWVz1N perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-aws-s3-202511121368-q6fav" deleted from scheduled-backup-32064 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-azure-blob-20251112131135-q6fav" deleted from scheduled-backup-32064 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-gcp-cs-2025111213945-q6fav" deleted from scheduled-backup-32064 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-minio-2025111213755-q6fav" deleted from scheduled-backup-32064 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-pvc-2025111213422-q6fav" deleted from scheduled-backup-32064 namespace + cat /tmp/tmp.qTemdLkKFS + rm /tmp/tmp.RVcPUWVz1N /tmp/tmp.qTemdLkKFS + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.sBnDEs5qGK ++ mktemp + local LAST_ERR=/tmp/tmp.v3udn4pJPP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sBnDEs5qGK No resources found + cat /tmp/tmp.v3udn4pJPP + rm /tmp/tmp.sBnDEs5qGK /tmp/tmp.v3udn4pJPP + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ++ mktemp + xargs kubectl delete ns ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.S8NLCuzAIa + awk '{print$1}' + local LAST_OUT=/tmp/tmp.k21asoV9fz ++ mktemp + local LAST_ERR=/tmp/tmp.HAjjlmSLq1 + local exit_status=0 ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.ShNxBmmVaV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.S8NLCuzAIa + cat /tmp/tmp.HAjjlmSLq1 + rm /tmp/tmp.S8NLCuzAIa /tmp/tmp.HAjjlmSLq1 + return 0 namespace "scheduled-backup-32064" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.k21asoV9fz namespace "pxc-operator" deleted + cat /tmp/tmp.ShNxBmmVaV + rm /tmp/tmp.k21asoV9fz /tmp/tmp.ShNxBmmVaV + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.IDLrYi20qF ++ mktemp + local LAST_ERR=/tmp/tmp.MGWJJy8nVr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IDLrYi20qF namespace/pxc-operator created + cat /tmp/tmp.MGWJJy8nVr + rm /tmp/tmp.IDLrYi20qF /tmp/tmp.MGWJJy8nVr + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.opCPCPLM5s +++ mktemp ++ local LAST_ERR=/tmp/tmp.C2mjouA3V2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.opCPCPLM5s ++ cat /tmp/tmp.C2mjouA3V2 ++ rm /tmp/tmp.opCPCPLM5s /tmp/tmp.C2mjouA3V2 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster6 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.hc22Dlwuoz ++ mktemp + local LAST_ERR=/tmp/tmp.H3XXsdYZCw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster6 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hc22Dlwuoz Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster6" modified. + cat /tmp/tmp.H3XXsdYZCw + rm /tmp/tmp.hc22Dlwuoz /tmp/tmp.H3XXsdYZCw + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.60Fdi3vSz8 ++ mktemp + local LAST_ERR=/tmp/tmp.xpmryIuQZe + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.60Fdi3vSz8 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.xpmryIuQZe + rm /tmp/tmp.60Fdi3vSz8 /tmp/tmp.xpmryIuQZe + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.HaIMD7rrSV ++ mktemp + local LAST_ERR=/tmp/tmp.GYrO1ksZDw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HaIMD7rrSV clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.GYrO1ksZDw + rm /tmp/tmp.HaIMD7rrSV /tmp/tmp.GYrO1ksZDw + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.PA64Kh7Ecv ++ mktemp + local LAST_ERR=/tmp/tmp.7k7KvZBT4Q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PA64Kh7Ecv deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.7k7KvZBT4Q + rm /tmp/tmp.PA64Kh7Ecv /tmp/tmp.7k7KvZBT4Q + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.qYBOxvvEsa ++ mktemp + local LAST_ERR=/tmp/tmp.UT696zbtSC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qYBOxvvEsa pod/percona-xtradb-cluster-operator-75f64b98f6-sfvrt condition met + cat /tmp/tmp.UT696zbtSC + rm /tmp/tmp.qYBOxvvEsa /tmp/tmp.UT696zbtSC + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.QPfk2OFNsw +++ mktemp ++ local LAST_ERR=/tmp/tmp.OEf7FeKQGw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QPfk2OFNsw ++ cat /tmp/tmp.OEf7FeKQGw ++ rm /tmp/tmp.QPfk2OFNsw /tmp/tmp.OEf7FeKQGw ++ return 0 + wait_pod percona-xtradb-cluster-operator-75f64b98f6-sfvrt 480 pxc-operator + local pod=percona-xtradb-cluster-operator-75f64b98f6-sfvrt + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-75f64b98f6-sfvrt ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-75f64b98f6-sfvrt condition met waiting for pod/percona-xtradb-cluster-operator-75f64b98f6-sfvrt to become Ready.Ok + sleep 3 + create_namespace scheduled-backup-29415 + local namespace=scheduled-backup-29415 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces scheduled-backup-29415' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces scheduled-backup-29415 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace scheduled-backup-29415 ++ mktemp + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.RcBKJg4hFI ++ mktemp + local LAST_OUT=/tmp/tmp.StwbQLslRp + xargs kubectl delete ns ++ mktemp + local LAST_ERR=/tmp/tmp.1zSSfKeg4v + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.cdHG3xYXfX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-29415 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.StwbQLslRp + cat /tmp/tmp.cdHG3xYXfX + rm /tmp/tmp.StwbQLslRp /tmp/tmp.cdHG3xYXfX + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-29415 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-29415 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.RcBKJg4hFI + cat /tmp/tmp.1zSSfKeg4v Error from server (NotFound): namespaces "scheduled-backup-29415" not found + rm /tmp/tmp.RcBKJg4hFI /tmp/tmp.1zSSfKeg4v + return 1 + : + wait_for_delete namespace/scheduled-backup-29415 + local res=namespace/scheduled-backup-29415 + echo -n 'waiting for namespace/scheduled-backup-29415 to be deleted' waiting for namespace/scheduled-backup-29415 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "scheduled-backup-29415" not found + desc 'create namespace scheduled-backup-29415' + set +o xtrace ----------------------------------------------------------------------------------- create namespace scheduled-backup-29415 ----------------------------------------------------------------------------------- + kubectl_bin create namespace scheduled-backup-29415 ++ mktemp + local LAST_OUT=/tmp/tmp.Rx4haedU59 ++ mktemp + local LAST_ERR=/tmp/tmp.yKERBVpKMv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace scheduled-backup-29415 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Rx4haedU59 namespace/scheduled-backup-29415 created + cat /tmp/tmp.yKERBVpKMv + rm /tmp/tmp.Rx4haedU59 /tmp/tmp.yKERBVpKMv + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.P2oM7q0TFT +++ mktemp ++ local LAST_ERR=/tmp/tmp.BGcecrhAIH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.P2oM7q0TFT ++ cat /tmp/tmp.BGcecrhAIH ++ rm /tmp/tmp.P2oM7q0TFT /tmp/tmp.BGcecrhAIH ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster6 --namespace=scheduled-backup-29415 ++ mktemp + local LAST_OUT=/tmp/tmp.ZwM58Q36zf ++ mktemp + local LAST_ERR=/tmp/tmp.bwEf3YLqhB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster6 --namespace=scheduled-backup-29415 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZwM58Q36zf Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster6" modified. + cat /tmp/tmp.bwEf3YLqhB + rm /tmp/tmp.ZwM58Q36zf /tmp/tmp.bwEf3YLqhB + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.zMQioAtNrp ++ mktemp + local LAST_ERR=/tmp/tmp.vOZr4priiK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zMQioAtNrp secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.vOZr4priiK + rm /tmp/tmp.zMQioAtNrp /tmp/tmp.vOZr4priiK + return 0 + start_minio + deploy_helm scheduled-backup-29415 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Wed Nov 12 13:20:01 2025 NAMESPACE: scheduled-backup-29415 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.scheduled-backup-29415.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-29415 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-29415 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-29415 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-29415 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sQypy8Rgb3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.6EjUgc0zay ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sQypy8Rgb3 ++ cat /tmp/tmp.6EjUgc0zay ++ rm /tmp/tmp.sQypy8Rgb3 /tmp/tmp.6EjUgc0zay ++ return 0 + MINIO_POD=minio-service-55fcc5d75f-jgctk + wait_pod minio-service-55fcc5d75f-jgctk + local pod=minio-service-55fcc5d75f-jgctk + local max_retry=480 + local ns= ++ echo minio-service-55fcc5d75f-jgctk ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minio-service-55fcc5d75f-jgctk condition met waiting for pod/minio-service-55fcc5d75f-jgctk to become Ready.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.TJUfaIzMLC ++ mktemp + local LAST_ERR=/tmp/tmp.ZlzI2GsF4M + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TJUfaIzMLC make_bucket: operator-testing pod "aws-cli" deleted from scheduled-backup-29415 namespace + cat /tmp/tmp.ZlzI2GsF4M + rm /tmp/tmp.TJUfaIzMLC /tmp/tmp.ZlzI2GsF4M + return 0 + cluster=scheduled-backup + cat - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.6zxPYgoEka ++ mktemp + local LAST_ERR=/tmp/tmp.D9VPG7CLPz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6zxPYgoEka priorityclass.scheduling.k8s.io/high-priority configured + cat /tmp/tmp.D9VPG7CLPz + rm /tmp/tmp.6zxPYgoEka /tmp/tmp.D9VPG7CLPz + return 0 + spinup_pxc scheduled-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local cluster=scheduled-backup + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Tgdar2DmVF ++ mktemp + local LAST_ERR=/tmp/tmp.HLBzq9MCjl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Tgdar2DmVF secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.HLBzq9MCjl + rm /tmp/tmp.Tgdar2DmVF /tmp/tmp.HLBzq9MCjl + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.pUXrYOzHST + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29415~ ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_ERR=/tmp/tmp.X2pyMBxkXa + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pUXrYOzHST deployment.apps/pxc-client created + cat /tmp/tmp.X2pyMBxkXa + rm /tmp/tmp.pUXrYOzHST /tmp/tmp.X2pyMBxkXa + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.6Ou92rNoOU + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + local LAST_ERR=/tmp/tmp.gERcSu5dRN + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ seq 0 2 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29415~ + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6Ou92rNoOU perconaxtradbcluster.pxc.percona.com/scheduled-backup created + cat /tmp/tmp.gERcSu5dRN + rm /tmp/tmp.6Ou92rNoOU /tmp/tmp.gERcSu5dRN + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy scheduled-backup ++ local target_cluster=scheduled-backup +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.E4my2eWOPx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MaxNf0oVKh +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.E4my2eWOPx +++ cat /tmp/tmp.MaxNf0oVKh +++ rm /tmp/tmp.E4my2eWOPx /tmp/tmp.MaxNf0oVKh +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ROsBcS5ddd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Xa4XSthWXm +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.ROsBcS5ddd +++ cat /tmp/tmp.Xa4XSthWXm +++ rm /tmp/tmp.ROsBcS5ddd /tmp/tmp.Xa4XSthWXm +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scheduled-backup-proxysql ++ return + local proxy=scheduled-backup-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-29415 ++ mktemp + local LAST_OUT=/tmp/tmp.trsNudfwen ++ mktemp + local LAST_ERR=/tmp/tmp.xF5lhmd4HN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-29415 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-29415 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-29415 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.trsNudfwen + cat /tmp/tmp.xF5lhmd4HN error: no matching resources found + rm /tmp/tmp.trsNudfwen /tmp/tmp.xF5lhmd4HN + return 1 + true + wait_for_running scheduled-backup-proxysql 1 + local name=scheduled-backup-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-proxysql-0 480 + local pod=scheduled-backup-proxysql-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-proxysql-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace pod/scheduled-backup-proxysql-0 condition met waiting for pod/scheduled-backup-proxysql-0 to become Ready.Ok + wait_for_running scheduled-backup-pxc 3 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 480 + local pod=scheduled-backup-pxc-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-0 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 480 + local pod=scheduled-backup-pxc-1 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 480 + local pod=scheduled-backup-pxc-2 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.rf0O1DzWBS +++ mktemp ++ local LAST_ERR=/tmp/tmp.w7ZCZ0yP8N ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rf0O1DzWBS ++ cat /tmp/tmp.w7ZCZ0yP8N ++ rm /tmp/tmp.rf0O1DzWBS /tmp/tmp.w7ZCZ0yP8N ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qX3lHeUW2p +++ mktemp ++ local LAST_ERR=/tmp/tmp.jLybFtnQJg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qX3lHeUW2p ++ cat /tmp/tmp.jLybFtnQJg Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.qX3lHeUW2p /tmp/tmp.jLybFtnQJg ++ return 0 + [[ -n '' ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SNlw2PlOkb +++ mktemp ++ local LAST_ERR=/tmp/tmp.gWs4GY9kLS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SNlw2PlOkb ++ cat /tmp/tmp.gWs4GY9kLS ++ rm /tmp/tmp.SNlw2PlOkb /tmp/tmp.gWs4GY9kLS ++ return 0 + client_pod=pxc-client-857d976497-tvfrw + wait_pod pxc-client-857d976497-tvfrw + local pod=pxc-client-857d976497-tvfrw + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-tvfrw ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-857d976497-tvfrw condition met waiting for pod/pxc-client-857d976497-tvfrw to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7rTRXB0ZKp +++ mktemp ++ local LAST_ERR=/tmp/tmp.tow2BOIRkT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7rTRXB0ZKp ++ cat /tmp/tmp.tow2BOIRkT ++ rm /tmp/tmp.7rTRXB0ZKp /tmp/tmp.tow2BOIRkT ++ return 0 + client_pod=pxc-client-857d976497-tvfrw + wait_pod pxc-client-857d976497-tvfrw + local pod=pxc-client-857d976497-tvfrw + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-tvfrw ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-857d976497-tvfrw condition met waiting for pod/pxc-client-857d976497-tvfrw to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9amxdwJlZe +++ mktemp ++ local LAST_ERR=/tmp/tmp.WLzdLjdG2q ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9amxdwJlZe ++ cat /tmp/tmp.WLzdLjdG2q ++ rm /tmp/tmp.9amxdwJlZe /tmp/tmp.WLzdLjdG2q ++ return 0 + client_pod=pxc-client-857d976497-tvfrw + wait_pod pxc-client-857d976497-tvfrw + local pod=pxc-client-857d976497-tvfrw + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-tvfrw ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-857d976497-tvfrw condition met waiting for pod/pxc-client-857d976497-tvfrw to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.kw8Clk3Y0a/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.kw8Clk3Y0a/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.b6fw5PtVpA +++ mktemp ++ local LAST_ERR=/tmp/tmp.5VUbYGlvJR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.b6fw5PtVpA ++ cat /tmp/tmp.5VUbYGlvJR ++ rm /tmp/tmp.b6fw5PtVpA /tmp/tmp.5VUbYGlvJR ++ return 0 + client_pod=pxc-client-857d976497-tvfrw + wait_pod pxc-client-857d976497-tvfrw + local pod=pxc-client-857d976497-tvfrw + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-857d976497-tvfrw + local container= + set +o xtrace pod/pxc-client-857d976497-tvfrw condition met waiting for pod/pxc-client-857d976497-tvfrw to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.kw8Clk3Y0a/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.kw8Clk3Y0a/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2TonCqv5Pd +++ mktemp ++ local LAST_ERR=/tmp/tmp.YGkjSJknAW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2TonCqv5Pd ++ cat /tmp/tmp.YGkjSJknAW ++ rm /tmp/tmp.2TonCqv5Pd /tmp/tmp.YGkjSJknAW ++ return 0 + client_pod=pxc-client-857d976497-tvfrw + wait_pod pxc-client-857d976497-tvfrw + local pod=pxc-client-857d976497-tvfrw + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-857d976497-tvfrw + local container= + set +o xtrace pod/pxc-client-857d976497-tvfrw condition met waiting for pod/pxc-client-857d976497-tvfrw to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.kw8Clk3Y0a/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.kw8Clk3Y0a/select-1.sql ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rgBZjUnrE6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.sU7D6tnPTN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rgBZjUnrE6 ++ cat /tmp/tmp.sU7D6tnPTN Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.rgBZjUnrE6 /tmp/tmp.sU7D6tnPTN ++ return 0 + '[' '' ']' + sleep 20 + desc 'add backups schedule for pvc storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for pvc storage ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.4IXw2MzOEM +++ mktemp ++ local LAST_ERR=/tmp/tmp.mSR08TgXct ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4IXw2MzOEM ++ cat /tmp/tmp.mSR08TgXct ++ rm /tmp/tmp.4IXw2MzOEM /tmp/tmp.mSR08TgXct ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster6 --namespace=scheduled-backup-29415 ++ mktemp + local LAST_OUT=/tmp/tmp.w9Yk9HZpeQ ++ mktemp + local LAST_ERR=/tmp/tmp.34YN6SWAkK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster6 --namespace=scheduled-backup-29415 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.w9Yk9HZpeQ Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster6" modified. + cat /tmp/tmp.34YN6SWAkK + rm /tmp/tmp.w9Yk9HZpeQ /tmp/tmp.34YN6SWAkK + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29415~ + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_OUT=/tmp/tmp.MwC47oXGz5 + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_ERR=/tmp/tmp.hJ4JUG6Xly + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MwC47oXGz5 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.hJ4JUG6Xly + rm /tmp/tmp.MwC47oXGz5 /tmp/tmp.hJ4JUG6Xly + return 0 + label_node ++ kubectl_bin get nodes --no-headers=true ++ grep -v master ++ head -n1 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wEH5iHr7YR +++ mktemp ++ local LAST_ERR=/tmp/tmp.jwTJwBUyGR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get nodes --no-headers=true ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wEH5iHr7YR ++ cat /tmp/tmp.jwTJwBUyGR ++ rm /tmp/tmp.wEH5iHr7YR /tmp/tmp.jwTJwBUyGR ++ return 0 + LABELED_NODE=gke-jen-pxc-2235-13a5e3e-default-pool-4ad54263-nl6t + kubectl_bin label nodes gke-jen-pxc-2235-13a5e3e-default-pool-4ad54263-nl6t backupWorker=True --overwrite ++ mktemp + local LAST_OUT=/tmp/tmp.yZbhbpBgnB ++ mktemp + local LAST_ERR=/tmp/tmp.r4TIFyJf7Q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label nodes gke-jen-pxc-2235-13a5e3e-default-pool-4ad54263-nl6t backupWorker=True --overwrite + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yZbhbpBgnB node/gke-jen-pxc-2235-13a5e3e-default-pool-4ad54263-nl6t not labeled + cat /tmp/tmp.r4TIFyJf7Q + rm /tmp/tmp.yZbhbpBgnB /tmp/tmp.r4TIFyJf7Q + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + local LAST_OUT=/tmp/tmp.f27iDD2Bw0 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29415~ + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + local LAST_ERR=/tmp/tmp.2HGFvqubGr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.f27iDD2Bw0 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.2HGFvqubGr + rm /tmp/tmp.f27iDD2Bw0 /tmp/tmp.2HGFvqubGr + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.OvKrKn38Tb +++ mktemp ++ local LAST_ERR=/tmp/tmp.vlUZ9psIHL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OvKrKn38Tb ++ cat /tmp/tmp.vlUZ9psIHL ++ rm /tmp/tmp.OvKrKn38Tb /tmp/tmp.vlUZ9psIHL ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.3cnsg924OR +++ mktemp ++ local LAST_ERR=/tmp/tmp.hAOansLZZx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3cnsg924OR ++ cat /tmp/tmp.hAOansLZZx ++ rm /tmp/tmp.3cnsg924OR /tmp/tmp.hAOansLZZx ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.wTL5T1eakl ++ mktemp + local LAST_ERR=/tmp/tmp.mi2dK8z0Di + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wTL5T1eakl NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-20251112132740-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112132740-q6fav Running 22s + cat /tmp/tmp.mi2dK8z0Di + rm /tmp/tmp.wTL5T1eakl /tmp/tmp.mi2dK8z0Di + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.XleSFIwkFa ++ mktemp + local LAST_ERR=/tmp/tmp.tPGosYKcrK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XleSFIwkFa NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-jgctk 1/1 Running 0 7m59s pxc-client-857d976497-tvfrw 2/2 Running 0 6m48s scheduled-backup-proxysql-0 3/3 Running 0 6m46s scheduled-backup-proxysql-1 3/3 Running 0 6m27s scheduled-backup-pxc-0 1/1 Running 0 6m46s scheduled-backup-pxc-1 1/1 Running 0 5m33s scheduled-backup-pxc-2 1/1 Running 0 4m19s xb-cron-scheduled-backup-pvc-20251112132740-q6fav-kqctf 1/1 Running 0 22s + cat /tmp/tmp.tPGosYKcrK + rm /tmp/tmp.XleSFIwkFa /tmp/tmp.tPGosYKcrK + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.tljKcWv5GK +++ mktemp ++ local LAST_ERR=/tmp/tmp.j0SLpXVPQB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tljKcWv5GK ++ cat /tmp/tmp.j0SLpXVPQB ++ rm /tmp/tmp.tljKcWv5GK /tmp/tmp.j0SLpXVPQB ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep :Failed ++ local LAST_OUT=/tmp/tmp.QOHLTPVJYB +++ mktemp ++ local LAST_ERR=/tmp/tmp.xMsUI8fgqj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QOHLTPVJYB ++ cat /tmp/tmp.xMsUI8fgqj ++ rm /tmp/tmp.QOHLTPVJYB /tmp/tmp.xMsUI8fgqj ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + local LAST_OUT=/tmp/tmp.w3AgHdBcDy + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29415~ ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_ERR=/tmp/tmp.2jUUENWJaN + local exit_status=0 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.w3AgHdBcDy perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.2jUUENWJaN + rm /tmp/tmp.w3AgHdBcDy /tmp/tmp.2jUUENWJaN + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29415~ + local LAST_OUT=/tmp/tmp.4iOvTZ5mlH ++ mktemp + local LAST_ERR=/tmp/tmp.VvIZ8LyecK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4iOvTZ5mlH perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.VvIZ8LyecK + rm /tmp/tmp.4iOvTZ5mlH /tmp/tmp.VvIZ8LyecK + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.5VgP4cd5Kh +++ mktemp ++ local LAST_ERR=/tmp/tmp.uVkimDDcYm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5VgP4cd5Kh ++ cat /tmp/tmp.uVkimDDcYm ++ rm /tmp/tmp.5VgP4cd5Kh /tmp/tmp.uVkimDDcYm ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kiP5hhjsKT +++ mktemp ++ local LAST_ERR=/tmp/tmp.4kgueyKsIl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kiP5hhjsKT ++ cat /tmp/tmp.4kgueyKsIl ++ rm /tmp/tmp.kiP5hhjsKT /tmp/tmp.4kgueyKsIl ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.1gBT1u000i ++ mktemp + local LAST_ERR=/tmp/tmp.gob54rgrDX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1gBT1u000i NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112132926-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-11-12-13:29:26-full Running 29s cron-scheduled-backup-pvc-20251112132740-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112132740-q6fav Succeeded 101s 2m15s + cat /tmp/tmp.gob54rgrDX + rm /tmp/tmp.1gBT1u000i /tmp/tmp.gob54rgrDX + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.UBFEJELZjM ++ mktemp + local LAST_ERR=/tmp/tmp.aKrNpgr5DB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UBFEJELZjM NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-jgctk 1/1 Running 0 9m52s pxc-client-857d976497-tvfrw 2/2 Running 0 8m41s scheduled-backup-proxysql-0 3/3 Running 0 8m39s scheduled-backup-proxysql-1 3/3 Running 0 8m20s scheduled-backup-pxc-0 1/1 Running 0 8m39s scheduled-backup-pxc-1 1/1 Running 0 7m26s scheduled-backup-pxc-2 1/1 Running 0 6m12s xb-cron-scheduled-backup-aws-s3-20251112132926-q6fav-5bpbs 1/1 Running 0 29s xb-cron-scheduled-backup-pvc-20251112132740-q6fav-kqctf 0/1 Completed 0 2m15s + cat /tmp/tmp.aKrNpgr5DB + rm /tmp/tmp.UBFEJELZjM /tmp/tmp.aKrNpgr5DB + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.cj3XWJteAC +++ mktemp ++ local LAST_ERR=/tmp/tmp.UNYXIzJbas ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cj3XWJteAC ++ cat /tmp/tmp.UNYXIzJbas ++ rm /tmp/tmp.cj3XWJteAC /tmp/tmp.UNYXIzJbas ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.UocR04l4pj +++ mktemp ++ local LAST_ERR=/tmp/tmp.5lJZhRwolx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UocR04l4pj ++ cat /tmp/tmp.5lJZhRwolx ++ rm /tmp/tmp.UocR04l4pj /tmp/tmp.5lJZhRwolx ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for minio storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for minio storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29415~ + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_OUT=/tmp/tmp.cxSfRRJ0PA ++ mktemp + local LAST_ERR=/tmp/tmp.bJRiDGnRJ6 + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cxSfRRJ0PA perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.bJRiDGnRJ6 + rm /tmp/tmp.cxSfRRJ0PA /tmp/tmp.bJRiDGnRJ6 + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29415~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.RlNuajlCLT ++ mktemp + local LAST_ERR=/tmp/tmp.Ql1F7zWFUo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RlNuajlCLT perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.Ql1F7zWFUo + rm /tmp/tmp.RlNuajlCLT /tmp/tmp.Ql1F7zWFUo + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.8j3d2aBbSU +++ mktemp ++ local LAST_ERR=/tmp/tmp.MLSkATaSoS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8j3d2aBbSU ++ cat /tmp/tmp.MLSkATaSoS ++ rm /tmp/tmp.8j3d2aBbSU /tmp/tmp.MLSkATaSoS ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.hCuerX5J9f +++ mktemp ++ local LAST_ERR=/tmp/tmp.T3EdwHdUY7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hCuerX5J9f ++ cat /tmp/tmp.T3EdwHdUY7 ++ rm /tmp/tmp.hCuerX5J9f /tmp/tmp.T3EdwHdUY7 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-1 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-2 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.UnYBTL6NDA ++ mktemp + local LAST_ERR=/tmp/tmp.8mP1QW1jDX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UnYBTL6NDA NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112132926-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-11-12-13:29:26-full Succeeded 95s 2m13s cron-scheduled-backup-minio-20251112133119-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-11-12-13:31:19-full Running 20s cron-scheduled-backup-pvc-20251112132740-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112132740-q6fav Succeeded 3m25s 3m59s + cat /tmp/tmp.8mP1QW1jDX + rm /tmp/tmp.UnYBTL6NDA /tmp/tmp.8mP1QW1jDX + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.JyTXURoKIl ++ mktemp + local LAST_ERR=/tmp/tmp.qhHW6Y14Hc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JyTXURoKIl NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-jgctk 1/1 Running 0 11m pxc-client-857d976497-tvfrw 2/2 Running 0 10m scheduled-backup-proxysql-0 3/3 Running 0 10m scheduled-backup-proxysql-1 3/3 Running 0 10m scheduled-backup-pxc-0 1/1 Running 0 10m scheduled-backup-pxc-1 1/1 Running 0 9m11s scheduled-backup-pxc-2 1/1 Running 0 7m57s xb-cron-scheduled-backup-aws-s3-20251112132926-q6fav-5bpbs 0/1 Completed 0 2m14s xb-cron-scheduled-backup-minio-20251112133119-q6fav-m554h 1/1 Running 0 21s xb-cron-scheduled-backup-pvc-20251112132740-q6fav-kqctf 0/1 Completed 0 4m + cat /tmp/tmp.qhHW6Y14Hc + rm /tmp/tmp.JyTXURoKIl /tmp/tmp.qhHW6Y14Hc + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.TTBkU9UkQO +++ mktemp ++ local LAST_ERR=/tmp/tmp.7f0VNx7YGa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TTBkU9UkQO ++ cat /tmp/tmp.7f0VNx7YGa ++ rm /tmp/tmp.TTBkU9UkQO /tmp/tmp.7f0VNx7YGa ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.uyogqg6GCf +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z5Me3ZPhND ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uyogqg6GCf ++ cat /tmp/tmp.Z5Me3ZPhND ++ rm /tmp/tmp.uyogqg6GCf /tmp/tmp.Z5Me3ZPhND ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for gcs storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for gcs storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29415~ + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + local LAST_OUT=/tmp/tmp.WvFMucqe2Y + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_ERR=/tmp/tmp.zmEezOuHUG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WvFMucqe2Y perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.zmEezOuHUG + rm /tmp/tmp.WvFMucqe2Y /tmp/tmp.zmEezOuHUG + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29415~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_OUT=/tmp/tmp.eYrJRcWQqm ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local LAST_ERR=/tmp/tmp.7M5aX68Hxe + local exit_status=0 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eYrJRcWQqm perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.7M5aX68Hxe + rm /tmp/tmp.eYrJRcWQqm /tmp/tmp.7M5aX68Hxe + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.5cYTsoZhEH +++ mktemp ++ local LAST_ERR=/tmp/tmp.fELMVbUkgL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5cYTsoZhEH ++ cat /tmp/tmp.fELMVbUkgL ++ rm /tmp/tmp.5cYTsoZhEH /tmp/tmp.fELMVbUkgL ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ grep :Failed ++ wc -l ++ local LAST_OUT=/tmp/tmp.OqGNPwSlPc +++ mktemp ++ local LAST_ERR=/tmp/tmp.KYYd6KyLjx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OqGNPwSlPc ++ cat /tmp/tmp.KYYd6KyLjx ++ rm /tmp/tmp.OqGNPwSlPc /tmp/tmp.KYYd6KyLjx ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.Ma94bQuIC6 ++ mktemp + local LAST_ERR=/tmp/tmp.fhzIXfAYCK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Ma94bQuIC6 NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112132926-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-11-12-13:29:26-full Succeeded 3m19s 3m57s cron-scheduled-backup-gcp-cs-2025111213335-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2025-11-12-13:33:05-full Running 18s cron-scheduled-backup-minio-20251112133119-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-11-12-13:31:19-full Succeeded 96s 2m4s cron-scheduled-backup-pvc-20251112132740-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112132740-q6fav Succeeded 5m9s 5m43s + cat /tmp/tmp.fhzIXfAYCK + rm /tmp/tmp.Ma94bQuIC6 /tmp/tmp.fhzIXfAYCK + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.sTZv0Jv0En ++ mktemp + local LAST_ERR=/tmp/tmp.5ku33SrW8R + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sTZv0Jv0En NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-jgctk 1/1 Running 0 13m pxc-client-857d976497-tvfrw 2/2 Running 0 12m scheduled-backup-proxysql-0 3/3 Running 0 12m scheduled-backup-proxysql-1 3/3 Running 0 11m scheduled-backup-pxc-0 1/1 Running 0 12m scheduled-backup-pxc-1 1/1 Running 0 10m scheduled-backup-pxc-2 1/1 Running 0 9m41s xb-cron-scheduled-backup-aws-s3-20251112132926-q6fav-5bpbs 0/1 Completed 0 3m58s xb-cron-scheduled-backup-gcp-cs-2025111213335-q6fav-59jvd 1/1 Running 0 19s xb-cron-scheduled-backup-minio-20251112133119-q6fav-m554h 0/1 Completed 0 2m5s xb-cron-scheduled-backup-pvc-20251112132740-q6fav-kqctf 0/1 Completed 0 5m44s + cat /tmp/tmp.5ku33SrW8R + rm /tmp/tmp.sTZv0Jv0En /tmp/tmp.5ku33SrW8R + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.BYUaSYzUj5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.m1VCFmhmbf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BYUaSYzUj5 ++ cat /tmp/tmp.m1VCFmhmbf ++ rm /tmp/tmp.BYUaSYzUj5 /tmp/tmp.m1VCFmhmbf ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.R2D6frW3a3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.tA4lvxTpZ6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.R2D6frW3a3 ++ cat /tmp/tmp.tA4lvxTpZ6 ++ rm /tmp/tmp.R2D6frW3a3 /tmp/tmp.tA4lvxTpZ6 ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for azure storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for azure storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + '[' -z '' ']' + kubectl_bin apply -f - ++ mktemp + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + local LAST_OUT=/tmp/tmp.IHaeynZxuO ++ mktemp + local LAST_ERR=/tmp/tmp.6uzhFTXUW4 + local exit_status=0 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29415~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IHaeynZxuO perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.6uzhFTXUW4 + rm /tmp/tmp.IHaeynZxuO /tmp/tmp.6uzhFTXUW4 + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + local LAST_OUT=/tmp/tmp.rCECqocKD8 ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + local LAST_ERR=/tmp/tmp.UWthwW1d81 + local exit_status=0 + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ seq 0 2 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29415~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rCECqocKD8 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.UWthwW1d81 + rm /tmp/tmp.rCECqocKD8 /tmp/tmp.UWthwW1d81 + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.faYZNu6Tps +++ mktemp ++ local LAST_ERR=/tmp/tmp.xqREYJ0RFM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.faYZNu6Tps ++ cat /tmp/tmp.xqREYJ0RFM ++ rm /tmp/tmp.faYZNu6Tps /tmp/tmp.xqREYJ0RFM ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.1bex6fIko6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.rnbPUxAeug ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1bex6fIko6 ++ cat /tmp/tmp.rnbPUxAeug ++ rm /tmp/tmp.1bex6fIko6 /tmp/tmp.rnbPUxAeug ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.t7eKuDYihj ++ mktemp + local LAST_ERR=/tmp/tmp.A4vfMtUMCt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.t7eKuDYihj NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112132926-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-11-12-13:29:26-full Succeeded 5m9s 5m47s cron-scheduled-backup-azure-blob-20251112133449-q6fav scheduled-backup azure-blob azure://operator-testing/scheduled-backup-2025-11-12-13:34:49-full Running 24s cron-scheduled-backup-gcp-cs-2025111213335-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2025-11-12-13:33:05-full Succeeded 93s 2m8s cron-scheduled-backup-minio-20251112133119-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-11-12-13:31:19-full Succeeded 3m26s 3m54s cron-scheduled-backup-pvc-20251112132740-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112132740-q6fav Succeeded 6m59s 7m33s + cat /tmp/tmp.A4vfMtUMCt + rm /tmp/tmp.t7eKuDYihj /tmp/tmp.A4vfMtUMCt + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.Ur7kjw0cLz ++ mktemp + local LAST_ERR=/tmp/tmp.l0t4nNpUPU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Ur7kjw0cLz NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-jgctk 1/1 Running 0 15m pxc-client-857d976497-tvfrw 2/2 Running 0 14m scheduled-backup-proxysql-0 3/3 Running 0 13m scheduled-backup-proxysql-1 3/3 Running 0 13m scheduled-backup-pxc-0 1/1 Running 0 13m scheduled-backup-pxc-1 1/1 Running 0 12m scheduled-backup-pxc-2 1/1 Running 0 11m xb-cron-scheduled-backup-aws-s3-20251112132926-q6fav-5bpbs 0/1 Completed 0 5m48s xb-cron-scheduled-backup-azure-blob-20251112133449-q6fav-xv8vr 1/1 Running 0 25s xb-cron-scheduled-backup-gcp-cs-2025111213335-q6fav-59jvd 0/1 Completed 0 2m9s xb-cron-scheduled-backup-minio-20251112133119-q6fav-m554h 0/1 Completed 0 3m55s xb-cron-scheduled-backup-pvc-20251112132740-q6fav-kqctf 0/1 Completed 0 7m34s + cat /tmp/tmp.l0t4nNpUPU + rm /tmp/tmp.Ur7kjw0cLz /tmp/tmp.l0t4nNpUPU + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.8iqUCXlWwP +++ mktemp ++ local LAST_ERR=/tmp/tmp.Y6AXiM3Mab ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8iqUCXlWwP ++ cat /tmp/tmp.Y6AXiM3Mab ++ rm /tmp/tmp.8iqUCXlWwP /tmp/tmp.Y6AXiM3Mab ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ grep :Failed ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.4oaazPwHP1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.91kRhgclpi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4oaazPwHP1 ++ cat /tmp/tmp.91kRhgclpi ++ rm /tmp/tmp.4oaazPwHP1 /tmp/tmp.91kRhgclpi ++ return 0 + [[ 0 -gt 0 ]] ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep pvc ++ cut -d: -f1 +++ mktemp ++ grep Succeeded ++ head -n1 ++ local LAST_OUT=/tmp/tmp.VeWkYZTtoL +++ mktemp ++ local LAST_ERR=/tmp/tmp.xGHfir0RFg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VeWkYZTtoL ++ cat /tmp/tmp.xGHfir0RFg ++ rm /tmp/tmp.VeWkYZTtoL /tmp/tmp.xGHfir0RFg ++ return 0 + FIRST_PVC_BACKUP=cron-scheduled-backup-pvc-20251112132740-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-pvc-20251112132740-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zVSVYJsaBW +++ mktemp ++ local LAST_ERR=/tmp/tmp.eVKRzkgF1c ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zVSVYJsaBW ++ cat /tmp/tmp.eVKRzkgF1c ++ rm /tmp/tmp.zVSVYJsaBW /tmp/tmp.eVKRzkgF1c ++ return 0 + JOB_PVC_BACKUP=xb-cron-scheduled-backup-pvc-20251112132740-q6fav ++ awk '{print $1}' ++ grep xb-cron-scheduled-backup-pvc-20251112132740 ++ kubectl_bin get pods +++ mktemp ++ local LAST_OUT=/tmp/tmp.ClpYzwq4PU +++ mktemp ++ local LAST_ERR=/tmp/tmp.1uLWdSunuu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ClpYzwq4PU ++ cat /tmp/tmp.1uLWdSunuu ++ rm /tmp/tmp.ClpYzwq4PU /tmp/tmp.1uLWdSunuu ++ return 0 + POD_PVC_BACKUP=xb-cron-scheduled-backup-pvc-20251112132740-q6fav-kqctf ++ grep Succeeded ++ cut -d: -f1 ++ head -n1 ++ grep minio ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wOMBszAj1a +++ mktemp ++ local LAST_ERR=/tmp/tmp.2haQuvqZEK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wOMBszAj1a ++ cat /tmp/tmp.2haQuvqZEK ++ rm /tmp/tmp.wOMBszAj1a /tmp/tmp.2haQuvqZEK ++ return 0 + FIRST_MINIO_BACKUP=cron-scheduled-backup-minio-20251112133119-q6fav ++ kubectl_bin get jobs ++ awk '{print $1}' +++ mktemp ++ grep cron-scheduled-backup-minio-20251112133119-q6fav ++ local LAST_OUT=/tmp/tmp.NloBwc5pFE +++ mktemp ++ local LAST_ERR=/tmp/tmp.KwWjtxi2rX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NloBwc5pFE ++ cat /tmp/tmp.KwWjtxi2rX ++ rm /tmp/tmp.NloBwc5pFE /tmp/tmp.KwWjtxi2rX ++ return 0 + JOB_MINIO_BACKUP=xb-cron-scheduled-backup-minio-20251112133119-q6fav ++ kubectl_bin get pods +++ mktemp ++ awk '{print $1}' ++ grep xb-cron-scheduled-backup-minio-20251112133119 ++ local LAST_OUT=/tmp/tmp.jew0hbrMRw +++ mktemp ++ local LAST_ERR=/tmp/tmp.gNY5RiZdPp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jew0hbrMRw ++ cat /tmp/tmp.gNY5RiZdPp ++ rm /tmp/tmp.jew0hbrMRw /tmp/tmp.gNY5RiZdPp ++ return 0 + POD_MINIO_BACKUP=xb-cron-scheduled-backup-minio-20251112133119-q6fav-m554h + '[' -z '' ']' ++ cut -d: -f1 ++ head -n1 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6RAdcOtJXK ++ grep Succeeded +++ mktemp ++ grep aws ++ local LAST_ERR=/tmp/tmp.f1RObxO12u ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6RAdcOtJXK ++ cat /tmp/tmp.f1RObxO12u ++ rm /tmp/tmp.6RAdcOtJXK /tmp/tmp.f1RObxO12u ++ return 0 + FIRST_AWS_BACKUP=cron-scheduled-backup-aws-s3-20251112132926-q6fav ++ kubectl_bin get jobs ++ awk '{print $1}' +++ mktemp ++ grep cron-scheduled-backup-aws-s3-20251112132926-q6fav ++ local LAST_OUT=/tmp/tmp.smN5mrNU9g +++ mktemp ++ local LAST_ERR=/tmp/tmp.qSrM2ZIe6c ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.smN5mrNU9g ++ cat /tmp/tmp.qSrM2ZIe6c ++ rm /tmp/tmp.smN5mrNU9g /tmp/tmp.qSrM2ZIe6c ++ return 0 + JOB_AWS_BACKUP=xb-cron-scheduled-backup-aws-s3-20251112132926-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-aws-s3-20251112132926 +++ mktemp ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.MnKnzOQ9zD +++ mktemp ++ local LAST_ERR=/tmp/tmp.NYlXQlE6x4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MnKnzOQ9zD ++ cat /tmp/tmp.NYlXQlE6x4 ++ rm /tmp/tmp.MnKnzOQ9zD /tmp/tmp.NYlXQlE6x4 ++ return 0 + POD_AWS_BACKUP=xb-cron-scheduled-backup-aws-s3-20251112132926-q6fav-5bpbs ++ grep gcp ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Hi3eck3VKc +++ mktemp ++ local LAST_ERR=/tmp/tmp.UmnOoFupR9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ head -n1 ++ grep Succeeded ++ cut -d: -f1 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Hi3eck3VKc ++ cat /tmp/tmp.UmnOoFupR9 ++ rm /tmp/tmp.Hi3eck3VKc /tmp/tmp.UmnOoFupR9 ++ return 0 + FIRST_GCP_BACKUP=cron-scheduled-backup-gcp-cs-2025111213335-q6fav ++ awk '{print $1}' ++ grep cron-scheduled-backup-gcp-cs-2025111213335-q6fav ++ kubectl_bin get jobs +++ mktemp ++ local LAST_OUT=/tmp/tmp.pnaDGbpJbs +++ mktemp ++ local LAST_ERR=/tmp/tmp.i5aYkRxcSp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pnaDGbpJbs ++ cat /tmp/tmp.i5aYkRxcSp ++ rm /tmp/tmp.pnaDGbpJbs /tmp/tmp.i5aYkRxcSp ++ return 0 + JOB_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-2025111213335-q6fav ++ awk '{print $1}' ++ grep xb-cron-scheduled-backup-gcp-cs-2025111213335 ++ kubectl_bin get pods +++ mktemp ++ local LAST_OUT=/tmp/tmp.gIsKeagG0T +++ mktemp ++ local LAST_ERR=/tmp/tmp.GJmi04MaeR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gIsKeagG0T ++ cat /tmp/tmp.GJmi04MaeR ++ rm /tmp/tmp.gIsKeagG0T /tmp/tmp.GJmi04MaeR ++ return 0 + POD_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-2025111213335-q6fav-59jvd ++ head -n1 ++ cut -d: -f1 ++ grep Succeeded ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep azure +++ mktemp ++ local LAST_OUT=/tmp/tmp.lqQG1wYf2b +++ mktemp ++ local LAST_ERR=/tmp/tmp.2oC2rclANw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lqQG1wYf2b ++ cat /tmp/tmp.2oC2rclANw ++ rm /tmp/tmp.lqQG1wYf2b /tmp/tmp.2oC2rclANw ++ return 0 + FIRST_AZURE_BACKUP=cron-scheduled-backup-azure-blob-20251112133449-q6fav ++ kubectl_bin get jobs +++ mktemp ++ local LAST_OUT=/tmp/tmp.IHabWb2sc0 ++ grep cron-scheduled-backup-azure-blob-20251112133449-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_ERR=/tmp/tmp.KuVGcRemF0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IHabWb2sc0 ++ cat /tmp/tmp.KuVGcRemF0 ++ rm /tmp/tmp.IHabWb2sc0 /tmp/tmp.KuVGcRemF0 ++ return 0 + JOB_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-20251112133449-q6fav ++ grep xb-cron-scheduled-backup-azure-blob-20251112133449 ++ awk '{print $1}' ++ kubectl_bin get pods +++ mktemp ++ local LAST_OUT=/tmp/tmp.JTcUM2tk7b +++ mktemp ++ local LAST_ERR=/tmp/tmp.owbMsqscvt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JTcUM2tk7b ++ cat /tmp/tmp.owbMsqscvt ++ rm /tmp/tmp.JTcUM2tk7b /tmp/tmp.owbMsqscvt ++ return 0 + POD_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-20251112133449-q6fav-xv8vr ++ kubectl_bin get pxc-backup cron-scheduled-backup-aws-s3-20251112132926-q6fav -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ cut -c 6- +++ mktemp ++ local LAST_OUT=/tmp/tmp.GdJsqh6Htt +++ mktemp ++ local LAST_ERR=/tmp/tmp.A0BcHNyY6u ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-aws-s3-20251112132926-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GdJsqh6Htt ++ cat /tmp/tmp.A0BcHNyY6u ++ rm /tmp/tmp.GdJsqh6Htt /tmp/tmp.A0BcHNyY6u ++ return 0 + BACKUP_DEST_AWS=operator-testing/scheduled-backup-2025-11-12-13:29:26-full ++ kubectl_bin get pxc-backup cron-scheduled-backup-gcp-cs-2025111213335-q6fav -o 'jsonpath={.status.destination}' ++ cut -c 6- ++ sed -e 's/.json$//' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vG16bCMesN +++ mktemp ++ local LAST_ERR=/tmp/tmp.OUfs4Xkmmj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-gcp-cs-2025111213335-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vG16bCMesN ++ cat /tmp/tmp.OUfs4Xkmmj ++ rm /tmp/tmp.vG16bCMesN /tmp/tmp.OUfs4Xkmmj ++ return 0 + BACKUP_DEST_GCP=operator-testing/scheduled-backup-2025-11-12-13:33:05-full ++ kubectl_bin get pxc-backup cron-scheduled-backup-azure-blob-20251112133449-q6fav -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ cut -c 9- +++ mktemp ++ local LAST_OUT=/tmp/tmp.Oc2BgumHDh +++ mktemp ++ local LAST_ERR=/tmp/tmp.QDYRK3wbQ8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-azure-blob-20251112133449-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Oc2BgumHDh ++ cat /tmp/tmp.QDYRK3wbQ8 ++ rm /tmp/tmp.Oc2BgumHDh /tmp/tmp.QDYRK3wbQ8 ++ return 0 + BACKUP_DEST_AZURE=operator-testing/scheduled-backup-2025-11-12-13:34:49-full + desc 'Check backup existence' + set +o xtrace ----------------------------------------------------------------------------------- Check backup existence ----------------------------------------------------------------------------------- + check_backup_existence https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 aws-s3 + path=https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + storage_name=aws-s3 + retry=0 ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 0 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 1 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 2 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 3 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 4 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 5 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 6 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 7 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 8 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 9 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 10 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 11 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 12 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 13 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 14 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 15 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 16 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 17 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 18 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 19 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 20 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 21 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 22 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 23 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 24 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 25 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 26 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 27 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 28 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 29 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:29:26-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 30 -ge 30 ']' + echo 'max retry count 30 reached. something went wrong with operator or kubernetes cluster' max retry count 30 reached. something went wrong with operator or kubernetes cluster + echo 'Backup was not found in bucket -- aws-s3' Backup was not found in bucket -- aws-s3 + exit 1