Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/logs/scheduled-backup-5-7.log Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 + main + create_infra scheduled-backup-20807 + local ns=scheduled-backup-20807 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n scheduled-backup-24087 scheduled-backup --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.4on7utu7iW ++ mktemp + local LAST_ERR=/tmp/tmp.Zs200fHHrT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4on7utu7iW perconaxtradbcluster.pxc.percona.com "scheduled-backup" deleted from scheduled-backup-24087 namespace + cat /tmp/tmp.Zs200fHHrT + rm /tmp/tmp.4on7utu7iW /tmp/tmp.Zs200fHHrT + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.baRRJ83ufo ++ mktemp + local LAST_ERR=/tmp/tmp.yJksGVfdXz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.baRRJ83ufo perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-aws-s3-2025111211031-q6fav" deleted from scheduled-backup-24087 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-azure-blob-202511121166-q6fav" deleted from scheduled-backup-24087 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-gcp-cs-2025111211417-q6fav" deleted from scheduled-backup-24087 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-minio-2025111211222-q6fav" deleted from scheduled-backup-24087 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-pvc-20251112105838-q6fav" deleted from scheduled-backup-24087 namespace + cat /tmp/tmp.yJksGVfdXz + rm /tmp/tmp.baRRJ83ufo /tmp/tmp.yJksGVfdXz + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.17Q79bIIqL ++ mktemp + local LAST_ERR=/tmp/tmp.3ewGLjcjdN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.17Q79bIIqL No resources found + cat /tmp/tmp.3ewGLjcjdN + rm /tmp/tmp.17Q79bIIqL /tmp/tmp.3ewGLjcjdN + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ awk '{print $1}' ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + awk '{print$1}' + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.ndSCKiYLIT ++ mktemp + local LAST_ERR=/tmp/tmp.025bvH3Keu + local exit_status=0 + kubectl_bin get ns ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.N7qBC1AzzA ++ mktemp + local LAST_ERR=/tmp/tmp.tgPjwBkRtm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.N7qBC1AzzA + cat /tmp/tmp.tgPjwBkRtm + rm /tmp/tmp.N7qBC1AzzA /tmp/tmp.tgPjwBkRtm + return 0 namespace "scheduled-backup-24087" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ndSCKiYLIT namespace "pxc-operator" deleted + cat /tmp/tmp.025bvH3Keu + rm /tmp/tmp.ndSCKiYLIT /tmp/tmp.025bvH3Keu + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.fIQaH4i8p9 ++ mktemp + local LAST_ERR=/tmp/tmp.BYoPDGSyFj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fIQaH4i8p9 namespace/pxc-operator created + cat /tmp/tmp.BYoPDGSyFj + rm /tmp/tmp.fIQaH4i8p9 /tmp/tmp.BYoPDGSyFj + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.PD3v3HrLyR +++ mktemp ++ local LAST_ERR=/tmp/tmp.BjWnbm72rP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PD3v3HrLyR ++ cat /tmp/tmp.BjWnbm72rP ++ rm /tmp/tmp.PD3v3HrLyR /tmp/tmp.BjWnbm72rP ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster2 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.EJD2Q4Ngfb ++ mktemp + local LAST_ERR=/tmp/tmp.IjWFKaINfr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster2 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EJD2Q4Ngfb Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster2" modified. + cat /tmp/tmp.IjWFKaINfr + rm /tmp/tmp.EJD2Q4Ngfb /tmp/tmp.IjWFKaINfr + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.AUL6WsZRhU ++ mktemp + local LAST_ERR=/tmp/tmp.FQeCBkLIbp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AUL6WsZRhU customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.FQeCBkLIbp + rm /tmp/tmp.AUL6WsZRhU /tmp/tmp.FQeCBkLIbp + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/cw-rbac.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.bHfxxKDaeh ++ mktemp + local LAST_ERR=/tmp/tmp.qF4m3yiiVX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bHfxxKDaeh clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.qF4m3yiiVX + rm /tmp/tmp.bHfxxKDaeh /tmp/tmp.qF4m3yiiVX + return 0 + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/cw-operator.yaml + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.3CdQbryM0s ++ mktemp + local LAST_ERR=/tmp/tmp.cEU0IgDg65 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3CdQbryM0s deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.cEU0IgDg65 + rm /tmp/tmp.3CdQbryM0s /tmp/tmp.cEU0IgDg65 + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.fPgvo1SOc9 ++ mktemp + local LAST_ERR=/tmp/tmp.ZGp0prFXlT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fPgvo1SOc9 pod/percona-xtradb-cluster-operator-b5f9c4897-qpl6l condition met + cat /tmp/tmp.ZGp0prFXlT + rm /tmp/tmp.fPgvo1SOc9 /tmp/tmp.ZGp0prFXlT + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.CnJ2A2EsIY +++ mktemp ++ local LAST_ERR=/tmp/tmp.T8UFOzpuMh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CnJ2A2EsIY ++ cat /tmp/tmp.T8UFOzpuMh ++ rm /tmp/tmp.CnJ2A2EsIY /tmp/tmp.T8UFOzpuMh ++ return 0 + wait_pod percona-xtradb-cluster-operator-b5f9c4897-qpl6l 480 pxc-operator + local pod=percona-xtradb-cluster-operator-b5f9c4897-qpl6l + local max_retry=480 + local ns=pxc-operator ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo percona-xtradb-cluster-operator-b5f9c4897-qpl6l + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-b5f9c4897-qpl6l condition met waiting for pod/percona-xtradb-cluster-operator-b5f9c4897-qpl6l to become Ready.Ok + sleep 3 + create_namespace scheduled-backup-20807 + local namespace=scheduled-backup-20807 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces scheduled-backup-20807' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces scheduled-backup-20807 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace scheduled-backup-20807 ++ mktemp + xargs kubectl delete ns + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.LLuto8FgbU ++ mktemp + local LAST_ERR=/tmp/tmp.d6DbpqJGw3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + local LAST_OUT=/tmp/tmp.Hcu7DmygCV ++ mktemp + local LAST_ERR=/tmp/tmp.OhBc0URRva + local exit_status=0 + awk '{print$1}' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-20807 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LLuto8FgbU + cat /tmp/tmp.d6DbpqJGw3 + rm /tmp/tmp.LLuto8FgbU /tmp/tmp.d6DbpqJGw3 + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-20807 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-20807 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.Hcu7DmygCV + cat /tmp/tmp.OhBc0URRva Error from server (NotFound): namespaces "scheduled-backup-20807" not found + rm /tmp/tmp.Hcu7DmygCV /tmp/tmp.OhBc0URRva + return 1 + : + wait_for_delete namespace/scheduled-backup-20807 + local res=namespace/scheduled-backup-20807 + echo -n 'waiting for namespace/scheduled-backup-20807 to be deleted' waiting for namespace/scheduled-backup-20807 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "scheduled-backup-20807" not found + desc 'create namespace scheduled-backup-20807' + set +o xtrace ----------------------------------------------------------------------------------- create namespace scheduled-backup-20807 ----------------------------------------------------------------------------------- + kubectl_bin create namespace scheduled-backup-20807 ++ mktemp + local LAST_OUT=/tmp/tmp.P2gLVFiF5c ++ mktemp + local LAST_ERR=/tmp/tmp.MEFsFlLWH9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace scheduled-backup-20807 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.P2gLVFiF5c namespace/scheduled-backup-20807 created + cat /tmp/tmp.MEFsFlLWH9 + rm /tmp/tmp.P2gLVFiF5c /tmp/tmp.MEFsFlLWH9 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.OJfpLhebww +++ mktemp ++ local LAST_ERR=/tmp/tmp.XhctTXhi1T ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OJfpLhebww ++ cat /tmp/tmp.XhctTXhi1T ++ rm /tmp/tmp.OJfpLhebww /tmp/tmp.XhctTXhi1T ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster2 --namespace=scheduled-backup-20807 ++ mktemp + local LAST_OUT=/tmp/tmp.2CAErw2v4V ++ mktemp + local LAST_ERR=/tmp/tmp.4FFI8thxE3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster2 --namespace=scheduled-backup-20807 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2CAErw2v4V Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster2" modified. + cat /tmp/tmp.4FFI8thxE3 + rm /tmp/tmp.2CAErw2v4V /tmp/tmp.4FFI8thxE3 + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.pWagc1SZ28 ++ mktemp + local LAST_ERR=/tmp/tmp.bbEIPFB7OH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pWagc1SZ28 secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.bbEIPFB7OH + rm /tmp/tmp.pWagc1SZ28 /tmp/tmp.bbEIPFB7OH + return 0 + start_minio + deploy_helm scheduled-backup-20807 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "hashicorp" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Wed Nov 12 11:14:37 2025 NAMESPACE: scheduled-backup-20807 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.scheduled-backup-20807.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-20807 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-20807 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-20807 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-20807 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.E29dcFw0En +++ mktemp ++ local LAST_ERR=/tmp/tmp.CpSSRnv2RV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.E29dcFw0En ++ cat /tmp/tmp.CpSSRnv2RV ++ rm /tmp/tmp.E29dcFw0En /tmp/tmp.CpSSRnv2RV ++ return 0 + MINIO_POD=minio-service-55fcc5d75f-pqkzc + wait_pod minio-service-55fcc5d75f-pqkzc + local pod=minio-service-55fcc5d75f-pqkzc + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo minio-service-55fcc5d75f-pqkzc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/minio-service-55fcc5d75f-pqkzc condition met waiting for pod/minio-service-55fcc5d75f-pqkzc to become Ready.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.v7LpEIEviO ++ mktemp + local LAST_ERR=/tmp/tmp.9tP3xGPTDU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.v7LpEIEviO make_bucket: operator-testing pod "aws-cli" deleted from scheduled-backup-20807 namespace + cat /tmp/tmp.9tP3xGPTDU + rm /tmp/tmp.v7LpEIEviO /tmp/tmp.9tP3xGPTDU + return 0 + cluster=scheduled-backup + kubectl_bin apply -f - + cat - ++ mktemp + local LAST_OUT=/tmp/tmp.DMKeC6JQPq ++ mktemp + local LAST_ERR=/tmp/tmp.MK9cIs1MJY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DMKeC6JQPq priorityclass.scheduling.k8s.io/high-priority configured + cat /tmp/tmp.MK9cIs1MJY + rm /tmp/tmp.DMKeC6JQPq /tmp/tmp.MK9cIs1MJY + return 0 + spinup_pxc scheduled-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local cluster=scheduled-backup + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.SKj3BTiUQa ++ mktemp + local LAST_ERR=/tmp/tmp.jIsoSmCoX0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SKj3BTiUQa secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.jIsoSmCoX0 + rm /tmp/tmp.SKj3BTiUQa /tmp/tmp.jIsoSmCoX0 + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_OUT=/tmp/tmp.cpfq7QMaaS + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-20807~ ++ mktemp + local LAST_ERR=/tmp/tmp.ZtIoC3ptZI + local exit_status=0 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cpfq7QMaaS deployment.apps/pxc-client created + cat /tmp/tmp.ZtIoC3ptZI + rm /tmp/tmp.cpfq7QMaaS /tmp/tmp.ZtIoC3ptZI + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-20807~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.jrEipMca9L ++ mktemp + local LAST_ERR=/tmp/tmp.JvRBUarkZO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jrEipMca9L perconaxtradbcluster.pxc.percona.com/scheduled-backup created + cat /tmp/tmp.JvRBUarkZO + rm /tmp/tmp.jrEipMca9L /tmp/tmp.JvRBUarkZO + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy scheduled-backup ++ local target_cluster=scheduled-backup +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kPeoAM9fr6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Bd2rdknsU2 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.kPeoAM9fr6 +++ cat /tmp/tmp.Bd2rdknsU2 +++ rm /tmp/tmp.kPeoAM9fr6 /tmp/tmp.Bd2rdknsU2 +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TskGxeDXjK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OdivcSE9Fo +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.TskGxeDXjK +++ cat /tmp/tmp.OdivcSE9Fo +++ rm /tmp/tmp.TskGxeDXjK /tmp/tmp.OdivcSE9Fo +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scheduled-backup-proxysql ++ return + local proxy=scheduled-backup-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-20807 ++ mktemp + local LAST_OUT=/tmp/tmp.764dtPG4Q6 ++ mktemp + local LAST_ERR=/tmp/tmp.rSkCQUTfsI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-20807 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-20807 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-20807 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.764dtPG4Q6 + cat /tmp/tmp.rSkCQUTfsI error: no matching resources found + rm /tmp/tmp.764dtPG4Q6 /tmp/tmp.rSkCQUTfsI + return 1 + true + wait_for_running scheduled-backup-proxysql 1 + local name=scheduled-backup-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-proxysql-0 480 + local pod=scheduled-backup-proxysql-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/scheduled-backup-proxysql-0 condition met waiting for pod/scheduled-backup-proxysql-0 to become Ready.Ok + wait_for_running scheduled-backup-pxc 3 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 480 + local pod=scheduled-backup-pxc-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 480 + local pod=scheduled-backup-pxc-1 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 480 + local pod=scheduled-backup-pxc-2 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.7gUyFbBbk0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.0bOy5b11Qc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7gUyFbBbk0 ++ cat /tmp/tmp.0bOy5b11Qc ++ rm /tmp/tmp.7gUyFbBbk0 /tmp/tmp.0bOy5b11Qc ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qfmtMu2Cl2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ah4PTfp2x7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qfmtMu2Cl2 ++ cat /tmp/tmp.Ah4PTfp2x7 Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.qfmtMu2Cl2 /tmp/tmp.Ah4PTfp2x7 ++ return 0 + [[ -n '' ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K6NSqxwCJ6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.351oAwlYsF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.K6NSqxwCJ6 ++ cat /tmp/tmp.351oAwlYsF ++ rm /tmp/tmp.K6NSqxwCJ6 /tmp/tmp.351oAwlYsF ++ return 0 + client_pod=pxc-client-857d976497-wwwhc + wait_pod pxc-client-857d976497-wwwhc + local pod=pxc-client-857d976497-wwwhc + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-857d976497-wwwhc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-857d976497-wwwhc condition met waiting for pod/pxc-client-857d976497-wwwhc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lreajUQkFd +++ mktemp ++ local LAST_ERR=/tmp/tmp.wrgwMblgij ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lreajUQkFd ++ cat /tmp/tmp.wrgwMblgij ++ rm /tmp/tmp.lreajUQkFd /tmp/tmp.wrgwMblgij ++ return 0 + client_pod=pxc-client-857d976497-wwwhc + wait_pod pxc-client-857d976497-wwwhc + local pod=pxc-client-857d976497-wwwhc + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-wwwhc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-857d976497-wwwhc condition met waiting for pod/pxc-client-857d976497-wwwhc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zRP9ZLdF2f +++ mktemp ++ local LAST_ERR=/tmp/tmp.XjP91qxIxv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zRP9ZLdF2f ++ cat /tmp/tmp.XjP91qxIxv ++ rm /tmp/tmp.zRP9ZLdF2f /tmp/tmp.XjP91qxIxv ++ return 0 + client_pod=pxc-client-857d976497-wwwhc + wait_pod pxc-client-857d976497-wwwhc + local pod=pxc-client-857d976497-wwwhc + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-wwwhc ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-857d976497-wwwhc condition met waiting for pod/pxc-client-857d976497-wwwhc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.o0DGGokT05/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.o0DGGokT05/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.y3TabKi17k +++ mktemp ++ local LAST_ERR=/tmp/tmp.gtPTzKwwVy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.y3TabKi17k ++ cat /tmp/tmp.gtPTzKwwVy ++ rm /tmp/tmp.y3TabKi17k /tmp/tmp.gtPTzKwwVy ++ return 0 + client_pod=pxc-client-857d976497-wwwhc + wait_pod pxc-client-857d976497-wwwhc + local pod=pxc-client-857d976497-wwwhc + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-857d976497-wwwhc ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-857d976497-wwwhc condition met waiting for pod/pxc-client-857d976497-wwwhc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.o0DGGokT05/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.o0DGGokT05/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yzEq0UQbyd +++ mktemp ++ local LAST_ERR=/tmp/tmp.KU6VzdMYO1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yzEq0UQbyd ++ cat /tmp/tmp.KU6VzdMYO1 ++ rm /tmp/tmp.yzEq0UQbyd /tmp/tmp.KU6VzdMYO1 ++ return 0 + client_pod=pxc-client-857d976497-wwwhc + wait_pod pxc-client-857d976497-wwwhc + local pod=pxc-client-857d976497-wwwhc + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-wwwhc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-857d976497-wwwhc condition met waiting for pod/pxc-client-857d976497-wwwhc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.o0DGGokT05/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.o0DGGokT05/select-1.sql ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5XoTcrRSQv +++ mktemp ++ egrep -o 'early-plugin-load=keyring_\w+.so' ++ local LAST_ERR=/tmp/tmp.uMV82ZhF4Y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5XoTcrRSQv ++ cat /tmp/tmp.uMV82ZhF4Y Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.5XoTcrRSQv /tmp/tmp.uMV82ZhF4Y ++ return 0 + '[' '' ']' + sleep 20 + desc 'add backups schedule for pvc storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for pvc storage ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.jCnJgSRHZD +++ mktemp ++ local LAST_ERR=/tmp/tmp.WPLz4RU27V ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jCnJgSRHZD ++ cat /tmp/tmp.WPLz4RU27V ++ rm /tmp/tmp.jCnJgSRHZD /tmp/tmp.WPLz4RU27V ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster2 --namespace=scheduled-backup-20807 ++ mktemp + local LAST_OUT=/tmp/tmp.PUFL6Wvm0F ++ mktemp + local LAST_ERR=/tmp/tmp.FpceCs7eLC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster2 --namespace=scheduled-backup-20807 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PUFL6Wvm0F Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster2" modified. + cat /tmp/tmp.FpceCs7eLC + rm /tmp/tmp.PUFL6Wvm0F /tmp/tmp.FpceCs7eLC + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-20807~ + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_OUT=/tmp/tmp.V9CcFQ4I6Z ++ mktemp + local LAST_ERR=/tmp/tmp.e1vye0vidS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.V9CcFQ4I6Z perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.e1vye0vidS + rm /tmp/tmp.V9CcFQ4I6Z /tmp/tmp.e1vye0vidS + return 0 + label_node ++ grep -v master ++ head -n1 ++ awk '{print $1}' ++ kubectl_bin get nodes --no-headers=true +++ mktemp ++ local LAST_OUT=/tmp/tmp.53dcvxbzhK +++ mktemp ++ local LAST_ERR=/tmp/tmp.c0e1PBkxh8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get nodes --no-headers=true ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.53dcvxbzhK ++ cat /tmp/tmp.c0e1PBkxh8 ++ rm /tmp/tmp.53dcvxbzhK /tmp/tmp.c0e1PBkxh8 ++ return 0 + LABELED_NODE=gke-jen-pxc-2234-269f369-default-pool-c62b03ba-8jlz + kubectl_bin label nodes gke-jen-pxc-2234-269f369-default-pool-c62b03ba-8jlz backupWorker=True --overwrite ++ mktemp + local LAST_OUT=/tmp/tmp.MALncHWi6X ++ mktemp + local LAST_ERR=/tmp/tmp.DNe3C63men + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label nodes gke-jen-pxc-2234-269f369-default-pool-c62b03ba-8jlz backupWorker=True --overwrite + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MALncHWi6X node/gke-jen-pxc-2234-269f369-default-pool-c62b03ba-8jlz not labeled + cat /tmp/tmp.DNe3C63men + rm /tmp/tmp.MALncHWi6X /tmp/tmp.DNe3C63men + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-20807~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.XwydSEX3ve ++ mktemp + local LAST_ERR=/tmp/tmp.2H35uHuYHs + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XwydSEX3ve perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.2H35uHuYHs + rm /tmp/tmp.XwydSEX3ve /tmp/tmp.2H35uHuYHs + return 0 + wait_all_backups ++ get_running_backups_amount ++ wc -l ++ grep -vE ':Succeeded|:Failed' ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K3OnLp5reQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.2ybC19kJia ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.K3OnLp5reQ ++ cat /tmp/tmp.2ybC19kJia ++ rm /tmp/tmp.K3OnLp5reQ /tmp/tmp.2ybC19kJia ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.4Gz3bd73L4 +++ mktemp ++ grep :Failed ++ local LAST_ERR=/tmp/tmp.LGQYyLAkiz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4Gz3bd73L4 ++ cat /tmp/tmp.LGQYyLAkiz ++ rm /tmp/tmp.4Gz3bd73L4 /tmp/tmp.LGQYyLAkiz ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.CPKzaCAHOn ++ mktemp + local LAST_ERR=/tmp/tmp.j1lPEFe3Xn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CPKzaCAHOn NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-20251112112224-q6fav scheduled-backup pvc Running 30s + cat /tmp/tmp.j1lPEFe3Xn + rm /tmp/tmp.CPKzaCAHOn /tmp/tmp.j1lPEFe3Xn + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.IzoFqBzHNK ++ mktemp + local LAST_ERR=/tmp/tmp.5qIBuBKo6S + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IzoFqBzHNK NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-pqkzc 1/1 Running 0 8m15s pxc-client-857d976497-wwwhc 2/2 Running 0 7m3s scheduled-backup-proxysql-0 3/3 Running 0 6m59s scheduled-backup-proxysql-1 3/3 Running 0 6m41s scheduled-backup-pxc-0 1/1 Running 0 6m59s scheduled-backup-pxc-1 1/1 Running 0 5m46s scheduled-backup-pxc-2 1/1 Running 0 4m33s xb-cron-scheduled-backup-pvc-20251112112224-q6fav-vsfnn 1/1 Running 0 31s + cat /tmp/tmp.5qIBuBKo6S + rm /tmp/tmp.IzoFqBzHNK /tmp/tmp.5qIBuBKo6S + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.2AdbLu9TdW +++ mktemp ++ local LAST_ERR=/tmp/tmp.oxjNUZRyd7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2AdbLu9TdW ++ cat /tmp/tmp.oxjNUZRyd7 ++ rm /tmp/tmp.2AdbLu9TdW /tmp/tmp.oxjNUZRyd7 ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.LvYqUt1PRm +++ mktemp ++ local LAST_ERR=/tmp/tmp.9qxHF1lLdL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LvYqUt1PRm ++ cat /tmp/tmp.9qxHF1lLdL ++ rm /tmp/tmp.LvYqUt1PRm /tmp/tmp.9qxHF1lLdL ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-20807~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_OUT=/tmp/tmp.10XjMCI8F2 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_ERR=/tmp/tmp.IWCBWhQ3QO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.10XjMCI8F2 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.IWCBWhQ3QO + rm /tmp/tmp.10XjMCI8F2 /tmp/tmp.IWCBWhQ3QO + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-20807~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_OUT=/tmp/tmp.MYwfsoLYcB + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + local LAST_ERR=/tmp/tmp.ZaMLN6l19D + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MYwfsoLYcB perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.ZaMLN6l19D + rm /tmp/tmp.MYwfsoLYcB /tmp/tmp.ZaMLN6l19D + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Wvii7iVaKX +++ mktemp ++ local LAST_ERR=/tmp/tmp.5rTc0i77Zc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Wvii7iVaKX ++ cat /tmp/tmp.5rTc0i77Zc ++ rm /tmp/tmp.Wvii7iVaKX /tmp/tmp.5rTc0i77Zc ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.c84VM2hSWB +++ mktemp ++ grep :Failed ++ wc -l ++ local LAST_ERR=/tmp/tmp.ZyllogtUmx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.c84VM2hSWB ++ cat /tmp/tmp.ZyllogtUmx ++ rm /tmp/tmp.c84VM2hSWB /tmp/tmp.ZyllogtUmx ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.rP1zKr3KDy ++ mktemp + local LAST_ERR=/tmp/tmp.5sbjPbxI4M + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rP1zKr3KDy NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112112419-q6fav scheduled-backup aws-s3 Running 22s cron-scheduled-backup-pvc-20251112112224-q6fav scheduled-backup pvc Succeeded 102s 2m17s + cat /tmp/tmp.5sbjPbxI4M + rm /tmp/tmp.rP1zKr3KDy /tmp/tmp.5sbjPbxI4M + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.4d08ceVpBP ++ mktemp + local LAST_ERR=/tmp/tmp.GpVCpfHco8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4d08ceVpBP NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-pqkzc 1/1 Running 0 10m pxc-client-857d976497-wwwhc 2/2 Running 0 8m49s scheduled-backup-proxysql-0 3/3 Running 0 8m45s scheduled-backup-proxysql-1 3/3 Running 0 8m27s scheduled-backup-pxc-0 1/1 Running 0 8m45s scheduled-backup-pxc-1 1/1 Running 0 7m32s scheduled-backup-pxc-2 1/1 Running 0 6m19s xb-cron-scheduled-backup-aws-s3-20251112112419-q6fav-f7vsr 1/1 Running 0 22s xb-cron-scheduled-backup-pvc-20251112112224-q6fav-vsfnn 0/1 Completed 0 2m17s + cat /tmp/tmp.GpVCpfHco8 + rm /tmp/tmp.4d08ceVpBP /tmp/tmp.GpVCpfHco8 + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.xyZ8rSa7k9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.fyPuuPHdus ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xyZ8rSa7k9 ++ cat /tmp/tmp.fyPuuPHdus ++ rm /tmp/tmp.xyZ8rSa7k9 /tmp/tmp.fyPuuPHdus ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.dj3esCE7YG +++ mktemp ++ local LAST_ERR=/tmp/tmp.aIg4GLQpOk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dj3esCE7YG ++ cat /tmp/tmp.aIg4GLQpOk ++ rm /tmp/tmp.dj3esCE7YG /tmp/tmp.aIg4GLQpOk ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for minio storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for minio storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.mwkMN1vQtE + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' ++ mktemp + local LAST_ERR=/tmp/tmp.5dBon6yZyb + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-20807~ + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mwkMN1vQtE perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.5dBon6yZyb + rm /tmp/tmp.mwkMN1vQtE /tmp/tmp.5dBon6yZyb + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.qJW0wFYvir ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local LAST_ERR=/tmp/tmp.JHxrUj9FMx + local exit_status=0 + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-20807~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qJW0wFYvir perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.JHxrUj9FMx + rm /tmp/tmp.qJW0wFYvir /tmp/tmp.JHxrUj9FMx + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.PWaWDTR6ap +++ mktemp ++ local LAST_ERR=/tmp/tmp.TYMUnsnp7i ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PWaWDTR6ap ++ cat /tmp/tmp.TYMUnsnp7i ++ rm /tmp/tmp.PWaWDTR6ap /tmp/tmp.TYMUnsnp7i ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.lDcLeYGJsB +++ mktemp ++ local LAST_ERR=/tmp/tmp.pZtQwb4KMW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lDcLeYGJsB ++ cat /tmp/tmp.pZtQwb4KMW ++ rm /tmp/tmp.lDcLeYGJsB /tmp/tmp.pZtQwb4KMW ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-0 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.xjncXT3rEC ++ mktemp + local LAST_ERR=/tmp/tmp.IZ0HsFdvG6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xjncXT3rEC NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112112419-q6fav scheduled-backup aws-s3 Succeeded 88s 2m6s cron-scheduled-backup-minio-2025111211265-q6fav scheduled-backup minio Running 20s cron-scheduled-backup-pvc-20251112112224-q6fav scheduled-backup pvc Succeeded 3m26s 4m1s + cat /tmp/tmp.IZ0HsFdvG6 + rm /tmp/tmp.xjncXT3rEC /tmp/tmp.IZ0HsFdvG6 + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.sY5w23waIm ++ mktemp + local LAST_ERR=/tmp/tmp.OS177iXbef + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sY5w23waIm NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-pqkzc 1/1 Running 0 11m pxc-client-857d976497-wwwhc 2/2 Running 0 10m scheduled-backup-proxysql-0 3/3 Running 0 10m scheduled-backup-proxysql-1 3/3 Running 0 10m scheduled-backup-pxc-0 1/1 Running 0 10m scheduled-backup-pxc-1 1/1 Running 0 9m17s scheduled-backup-pxc-2 1/1 Running 0 8m4s xb-cron-scheduled-backup-aws-s3-20251112112419-q6fav-f7vsr 0/1 Completed 0 2m7s xb-cron-scheduled-backup-minio-2025111211265-q6fav-ngn48 1/1 Running 0 21s xb-cron-scheduled-backup-pvc-20251112112224-q6fav-vsfnn 0/1 Completed 0 4m2s + cat /tmp/tmp.OS177iXbef + rm /tmp/tmp.sY5w23waIm /tmp/tmp.OS177iXbef + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.VfRE8Rc6mR +++ mktemp ++ local LAST_ERR=/tmp/tmp.3dP03Tvhc2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VfRE8Rc6mR ++ cat /tmp/tmp.3dP03Tvhc2 ++ rm /tmp/tmp.VfRE8Rc6mR /tmp/tmp.3dP03Tvhc2 ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.HPbPgZxhUq +++ mktemp ++ local LAST_ERR=/tmp/tmp.sTQ3qj7CId ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HPbPgZxhUq ++ cat /tmp/tmp.sTQ3qj7CId ++ rm /tmp/tmp.HPbPgZxhUq /tmp/tmp.sTQ3qj7CId ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for gcs storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for gcs storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-20807~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_OUT=/tmp/tmp.0mPEPhyaV2 ++ mktemp + local LAST_ERR=/tmp/tmp.IJNAL1Ylhl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0mPEPhyaV2 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.IJNAL1Ylhl + rm /tmp/tmp.0mPEPhyaV2 /tmp/tmp.IJNAL1Ylhl + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-20807~ + local LAST_OUT=/tmp/tmp.mLXeH0SsEX + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.1MPnonRCXC + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mLXeH0SsEX perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.1MPnonRCXC + rm /tmp/tmp.mLXeH0SsEX /tmp/tmp.1MPnonRCXC + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZqKwN1LbOn +++ mktemp ++ local LAST_ERR=/tmp/tmp.ftw4Nl5aPR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZqKwN1LbOn ++ cat /tmp/tmp.ftw4Nl5aPR ++ rm /tmp/tmp.ZqKwN1LbOn /tmp/tmp.ftw4Nl5aPR ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ grep :Failed ++ local LAST_OUT=/tmp/tmp.HxbaRUMJ8L +++ mktemp ++ local LAST_ERR=/tmp/tmp.TDogDzGVLR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HxbaRUMJ8L ++ cat /tmp/tmp.TDogDzGVLR ++ rm /tmp/tmp.HxbaRUMJ8L /tmp/tmp.TDogDzGVLR ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.To5L3ZGTxp ++ mktemp + local LAST_ERR=/tmp/tmp.dSCztxrKWM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.To5L3ZGTxp NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112112419-q6fav scheduled-backup aws-s3 Succeeded 3m11s 3m49s cron-scheduled-backup-gcp-cs-20251112112750-q6fav scheduled-backup gcp-cs Running 18s cron-scheduled-backup-minio-2025111211265-q6fav scheduled-backup minio Succeeded 95s 2m3s cron-scheduled-backup-pvc-20251112112224-q6fav scheduled-backup pvc Succeeded 5m9s 5m44s + cat /tmp/tmp.dSCztxrKWM + rm /tmp/tmp.To5L3ZGTxp /tmp/tmp.dSCztxrKWM + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.cJaa8t090a ++ mktemp + local LAST_ERR=/tmp/tmp.9mQTn8uWpU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cJaa8t090a NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-pqkzc 1/1 Running 0 13m pxc-client-857d976497-wwwhc 2/2 Running 0 12m scheduled-backup-proxysql-0 3/3 Running 0 12m scheduled-backup-proxysql-1 3/3 Running 0 11m scheduled-backup-pxc-0 1/1 Running 0 12m scheduled-backup-pxc-1 1/1 Running 0 11m scheduled-backup-pxc-2 1/1 Running 0 9m47s xb-cron-scheduled-backup-aws-s3-20251112112419-q6fav-f7vsr 0/1 Completed 0 3m50s xb-cron-scheduled-backup-gcp-cs-20251112112750-q6fav-jbcs8 1/1 Running 0 19s xb-cron-scheduled-backup-minio-2025111211265-q6fav-ngn48 0/1 Completed 0 2m4s xb-cron-scheduled-backup-pvc-20251112112224-q6fav-vsfnn 0/1 Completed 0 5m45s + cat /tmp/tmp.9mQTn8uWpU + rm /tmp/tmp.cJaa8t090a /tmp/tmp.9mQTn8uWpU + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.B8XAxEGTwh +++ mktemp ++ local LAST_ERR=/tmp/tmp.HQOrdz5SRq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.B8XAxEGTwh ++ cat /tmp/tmp.HQOrdz5SRq ++ rm /tmp/tmp.B8XAxEGTwh /tmp/tmp.HQOrdz5SRq ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.wv8FmZ1qYH +++ mktemp ++ local LAST_ERR=/tmp/tmp.DuS4YaTZLn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wv8FmZ1qYH ++ cat /tmp/tmp.DuS4YaTZLn ++ rm /tmp/tmp.wv8FmZ1qYH /tmp/tmp.DuS4YaTZLn ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for azure storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for azure storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.tDFYNuZGpU + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-20807~ + local LAST_ERR=/tmp/tmp.yItU6MnfWV + local exit_status=0 + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ seq 0 2 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tDFYNuZGpU perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.yItU6MnfWV + rm /tmp/tmp.tDFYNuZGpU /tmp/tmp.yItU6MnfWV + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.9W5LTnQzEO + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-20807~ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_ERR=/tmp/tmp.Nnt3O5qBPS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9W5LTnQzEO perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.Nnt3O5qBPS + rm /tmp/tmp.9W5LTnQzEO /tmp/tmp.Nnt3O5qBPS + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gCM1ULiJfA +++ mktemp ++ local LAST_ERR=/tmp/tmp.ECXvCA8NJc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gCM1ULiJfA ++ cat /tmp/tmp.ECXvCA8NJc ++ rm /tmp/tmp.gCM1ULiJfA /tmp/tmp.ECXvCA8NJc ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.mwEL4K1GJx +++ mktemp ++ local LAST_ERR=/tmp/tmp.J8OrytoXiC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mwEL4K1GJx ++ cat /tmp/tmp.J8OrytoXiC ++ rm /tmp/tmp.mwEL4K1GJx /tmp/tmp.J8OrytoXiC ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.ofuMW3ZR5a ++ mktemp + local LAST_ERR=/tmp/tmp.cFTUkPiZQx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ofuMW3ZR5a NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112112419-q6fav scheduled-backup aws-s3 Succeeded 5m1s 5m39s cron-scheduled-backup-azure-blob-20251112112933-q6fav scheduled-backup azure-blob Running 25s cron-scheduled-backup-gcp-cs-20251112112750-q6fav scheduled-backup gcp-cs Succeeded 93s 2m8s cron-scheduled-backup-minio-2025111211265-q6fav scheduled-backup minio Succeeded 3m25s 3m53s cron-scheduled-backup-pvc-20251112112224-q6fav scheduled-backup pvc Succeeded 6m59s 7m34s + cat /tmp/tmp.cFTUkPiZQx + rm /tmp/tmp.ofuMW3ZR5a /tmp/tmp.cFTUkPiZQx + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.AZ1EM8pb4T ++ mktemp + local LAST_ERR=/tmp/tmp.ggtpCPi14U + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AZ1EM8pb4T NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-pqkzc 1/1 Running 0 15m pxc-client-857d976497-wwwhc 2/2 Running 0 14m scheduled-backup-proxysql-0 3/3 Running 0 14m scheduled-backup-proxysql-1 3/3 Running 0 13m scheduled-backup-pxc-0 1/1 Running 0 14m scheduled-backup-pxc-1 1/1 Running 0 12m scheduled-backup-pxc-2 1/1 Running 0 11m xb-cron-scheduled-backup-aws-s3-20251112112419-q6fav-f7vsr 0/1 Completed 0 5m40s xb-cron-scheduled-backup-azure-blob-20251112112933-q6fav-5pw8g 1/1 Running 0 26s xb-cron-scheduled-backup-gcp-cs-20251112112750-q6fav-jbcs8 0/1 Completed 0 2m9s xb-cron-scheduled-backup-minio-2025111211265-q6fav-ngn48 0/1 Completed 0 3m54s xb-cron-scheduled-backup-pvc-20251112112224-q6fav-vsfnn 0/1 Completed 0 7m35s + cat /tmp/tmp.ggtpCPi14U + rm /tmp/tmp.AZ1EM8pb4T /tmp/tmp.ggtpCPi14U + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ka4cIy4I4j +++ mktemp ++ local LAST_ERR=/tmp/tmp.iP2vFbkH1u ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ka4cIy4I4j ++ cat /tmp/tmp.iP2vFbkH1u ++ rm /tmp/tmp.Ka4cIy4I4j /tmp/tmp.iP2vFbkH1u ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ grep :Failed ++ local LAST_OUT=/tmp/tmp.G5OjbyVe4I +++ mktemp ++ local LAST_ERR=/tmp/tmp.LDJswlsMWw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.G5OjbyVe4I ++ cat /tmp/tmp.LDJswlsMWw ++ rm /tmp/tmp.G5OjbyVe4I /tmp/tmp.LDJswlsMWw ++ return 0 + [[ 0 -gt 0 ]] ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ head -n1 +++ mktemp ++ cut -d: -f1 ++ grep pvc ++ local LAST_OUT=/tmp/tmp.mOJ073ZeFy +++ mktemp ++ local LAST_ERR=/tmp/tmp.HpJqgUl9Ym ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mOJ073ZeFy ++ cat /tmp/tmp.HpJqgUl9Ym ++ rm /tmp/tmp.mOJ073ZeFy /tmp/tmp.HpJqgUl9Ym ++ return 0 + FIRST_PVC_BACKUP=cron-scheduled-backup-pvc-20251112112224-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-pvc-20251112112224-q6fav +++ mktemp ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.eOoQ4OLEPL +++ mktemp ++ local LAST_ERR=/tmp/tmp.NLzDYljM9F ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eOoQ4OLEPL ++ cat /tmp/tmp.NLzDYljM9F ++ rm /tmp/tmp.eOoQ4OLEPL /tmp/tmp.NLzDYljM9F ++ return 0 + JOB_PVC_BACKUP=xb-cron-scheduled-backup-pvc-20251112112224-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-pvc-20251112112224 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.60IUbkUjqU +++ mktemp ++ local LAST_ERR=/tmp/tmp.dEhdRxbkJa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.60IUbkUjqU ++ cat /tmp/tmp.dEhdRxbkJa ++ rm /tmp/tmp.60IUbkUjqU /tmp/tmp.dEhdRxbkJa ++ return 0 + POD_PVC_BACKUP=xb-cron-scheduled-backup-pvc-20251112112224-q6fav-vsfnn ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded +++ mktemp ++ cut -d: -f1 ++ head -n1 ++ grep minio ++ local LAST_OUT=/tmp/tmp.ee2ddvbkec +++ mktemp ++ local LAST_ERR=/tmp/tmp.WoGtP7VRWs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ee2ddvbkec ++ cat /tmp/tmp.WoGtP7VRWs ++ rm /tmp/tmp.ee2ddvbkec /tmp/tmp.WoGtP7VRWs ++ return 0 + FIRST_MINIO_BACKUP=cron-scheduled-backup-minio-2025111211265-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-minio-2025111211265-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.m4TwJbnoGr +++ mktemp ++ local LAST_ERR=/tmp/tmp.8TQRBrWlVJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.m4TwJbnoGr ++ cat /tmp/tmp.8TQRBrWlVJ ++ rm /tmp/tmp.m4TwJbnoGr /tmp/tmp.8TQRBrWlVJ ++ return 0 + JOB_MINIO_BACKUP=xb-cron-scheduled-backup-minio-2025111211265-q6fav ++ kubectl_bin get pods ++ awk '{print $1}' +++ mktemp ++ grep xb-cron-scheduled-backup-minio-2025111211265 ++ local LAST_OUT=/tmp/tmp.xtVJSyb2Na +++ mktemp ++ local LAST_ERR=/tmp/tmp.pptbL7meWC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xtVJSyb2Na ++ cat /tmp/tmp.pptbL7meWC ++ rm /tmp/tmp.xtVJSyb2Na /tmp/tmp.pptbL7meWC ++ return 0 + POD_MINIO_BACKUP=xb-cron-scheduled-backup-minio-2025111211265-q6fav-ngn48 + '[' -z '' ']' ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ grep aws ++ head -n1 ++ cut -d: -f1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.ubymZeZpto +++ mktemp ++ local LAST_ERR=/tmp/tmp.X52DlpBo2c ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ubymZeZpto ++ cat /tmp/tmp.X52DlpBo2c ++ rm /tmp/tmp.ubymZeZpto /tmp/tmp.X52DlpBo2c ++ return 0 + FIRST_AWS_BACKUP=cron-scheduled-backup-aws-s3-20251112112419-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-aws-s3-20251112112419-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KEWEluNKAP +++ mktemp ++ local LAST_ERR=/tmp/tmp.sGBKInlLI1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KEWEluNKAP ++ cat /tmp/tmp.sGBKInlLI1 ++ rm /tmp/tmp.KEWEluNKAP /tmp/tmp.sGBKInlLI1 ++ return 0 + JOB_AWS_BACKUP=xb-cron-scheduled-backup-aws-s3-20251112112419-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-aws-s3-20251112112419 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pKUJmuHuqj +++ mktemp ++ local LAST_ERR=/tmp/tmp.9XAjjZYyVm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pKUJmuHuqj ++ cat /tmp/tmp.9XAjjZYyVm ++ rm /tmp/tmp.pKUJmuHuqj /tmp/tmp.9XAjjZYyVm ++ return 0 + POD_AWS_BACKUP=xb-cron-scheduled-backup-aws-s3-20251112112419-q6fav-f7vsr ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ grep gcp ++ cut -d: -f1 ++ head -n1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.wmSvdmnKva +++ mktemp ++ local LAST_ERR=/tmp/tmp.iRjvHk6Nr5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wmSvdmnKva ++ cat /tmp/tmp.iRjvHk6Nr5 ++ rm /tmp/tmp.wmSvdmnKva /tmp/tmp.iRjvHk6Nr5 ++ return 0 + FIRST_GCP_BACKUP=cron-scheduled-backup-gcp-cs-20251112112750-q6fav ++ kubectl_bin get jobs ++ awk '{print $1}' +++ mktemp ++ grep cron-scheduled-backup-gcp-cs-20251112112750-q6fav ++ local LAST_OUT=/tmp/tmp.K7KVoZKpK9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.cFUQe2SgRK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.K7KVoZKpK9 ++ cat /tmp/tmp.cFUQe2SgRK ++ rm /tmp/tmp.K7KVoZKpK9 /tmp/tmp.cFUQe2SgRK ++ return 0 + JOB_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-20251112112750-q6fav ++ kubectl_bin get pods ++ awk '{print $1}' ++ grep xb-cron-scheduled-backup-gcp-cs-20251112112750 +++ mktemp ++ local LAST_OUT=/tmp/tmp.INFdKs87al +++ mktemp ++ local LAST_ERR=/tmp/tmp.lwwB5ZxVdX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.INFdKs87al ++ cat /tmp/tmp.lwwB5ZxVdX ++ rm /tmp/tmp.INFdKs87al /tmp/tmp.lwwB5ZxVdX ++ return 0 + POD_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-20251112112750-q6fav-jbcs8 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep azure ++ head -n1 ++ cut -d: -f1 ++ grep Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.i9OpzvQFcp +++ mktemp ++ local LAST_ERR=/tmp/tmp.PcnD8o4vQH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.i9OpzvQFcp ++ cat /tmp/tmp.PcnD8o4vQH ++ rm /tmp/tmp.i9OpzvQFcp /tmp/tmp.PcnD8o4vQH ++ return 0 + FIRST_AZURE_BACKUP=cron-scheduled-backup-azure-blob-20251112112933-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-azure-blob-20251112112933-q6fav +++ mktemp ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.h1gSyperFP +++ mktemp ++ local LAST_ERR=/tmp/tmp.CYBgDmccZy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.h1gSyperFP ++ cat /tmp/tmp.CYBgDmccZy ++ rm /tmp/tmp.h1gSyperFP /tmp/tmp.CYBgDmccZy ++ return 0 + JOB_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-20251112112933-q6fav ++ kubectl_bin get pods ++ awk '{print $1}' ++ grep xb-cron-scheduled-backup-azure-blob-20251112112933 +++ mktemp ++ local LAST_OUT=/tmp/tmp.SzQNeMenPZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.1IKXL4xev9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SzQNeMenPZ ++ cat /tmp/tmp.1IKXL4xev9 ++ rm /tmp/tmp.SzQNeMenPZ /tmp/tmp.1IKXL4xev9 ++ return 0 + POD_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-20251112112933-q6fav-5pw8g ++ kubectl_bin get pxc-backup cron-scheduled-backup-aws-s3-20251112112419-q6fav -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' +++ mktemp ++ cut -c 6- ++ local LAST_OUT=/tmp/tmp.a0AUUVBVbS +++ mktemp ++ local LAST_ERR=/tmp/tmp.G6yRb9Cw5l ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-aws-s3-20251112112419-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.a0AUUVBVbS ++ cat /tmp/tmp.G6yRb9Cw5l ++ rm /tmp/tmp.a0AUUVBVbS /tmp/tmp.G6yRb9Cw5l ++ return 0 + BACKUP_DEST_AWS= ++ kubectl_bin get pxc-backup cron-scheduled-backup-gcp-cs-20251112112750-q6fav -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ cut -c 6- +++ mktemp ++ local LAST_OUT=/tmp/tmp.eJd27dclRa +++ mktemp ++ local LAST_ERR=/tmp/tmp.FGBqazMTeg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-gcp-cs-20251112112750-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eJd27dclRa ++ cat /tmp/tmp.FGBqazMTeg ++ rm /tmp/tmp.eJd27dclRa /tmp/tmp.FGBqazMTeg ++ return 0 + BACKUP_DEST_GCP= ++ kubectl_bin get pxc-backup cron-scheduled-backup-azure-blob-20251112112933-q6fav -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kMgNlECGGb +++ mktemp ++ local LAST_ERR=/tmp/tmp.iFeCnTc3gM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-azure-blob-20251112112933-q6fav -o 'jsonpath={.status.destination}' ++ cut -c 9- ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kMgNlECGGb ++ cat /tmp/tmp.iFeCnTc3gM ++ rm /tmp/tmp.kMgNlECGGb /tmp/tmp.iFeCnTc3gM ++ return 0 + BACKUP_DEST_AZURE= + desc 'Check backup existence' + set +o xtrace ----------------------------------------------------------------------------------- Check backup existence ----------------------------------------------------------------------------------- + check_backup_existence https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 aws-s3 + path=https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + storage_name=aws-s3 + retry=0 ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 0 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 1 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 2 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 3 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 4 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 5 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 6 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 7 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 8 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 9 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 10 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 11 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 12 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 13 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 14 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 15 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 16 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 17 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 18 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 19 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 20 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 21 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 22 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 23 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 24 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 25 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 26 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 27 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 28 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 29 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/.sst_info/sst_info.00000000000000000000 + [[ 404 -eq 200 ]] + '[' 30 -ge 30 ']' + echo 'max retry count 30 reached. something went wrong with operator or kubernetes cluster' max retry count 30 reached. something went wrong with operator or kubernetes cluster + echo 'Backup was not found in bucket -- aws-s3' Backup was not found in bucket -- aws-s3 + exit 1