Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/logs/scheduled-backup-8-0.log Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 + main + create_infra scheduled-backup-25427 + local ns=scheduled-backup-25427 + '[' -n pxc-operator ']' + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl get pxc --all-namespaces -o wide + kubectl patch pxc -n scheduled-backup-26591 scheduled-backup --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.GiA4WWTFp3 ++ mktemp + local LAST_ERR=/tmp/tmp.Tiu2B1HXkn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GiA4WWTFp3 perconaxtradbcluster.pxc.percona.com "scheduled-backup" deleted from scheduled-backup-26591 namespace + cat /tmp/tmp.Tiu2B1HXkn + rm /tmp/tmp.GiA4WWTFp3 /tmp/tmp.Tiu2B1HXkn + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.HMSfkr9Tic ++ mktemp + local LAST_ERR=/tmp/tmp.rc3aXZobvU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HMSfkr9Tic perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-aws-s3-2025111213958-q6fav" deleted from scheduled-backup-26591 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-azure-blob-20251112131527-q6fav" deleted from scheduled-backup-26591 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-gcp-cs-20251112131337-q6fav" deleted from scheduled-backup-26591 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-minio-20251112131148-q6fav" deleted from scheduled-backup-26591 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-pvc-202511121383-q6fav" deleted from scheduled-backup-26591 namespace + cat /tmp/tmp.rc3aXZobvU + rm /tmp/tmp.HMSfkr9Tic /tmp/tmp.rc3aXZobvU + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.5C4VyMwFR2 ++ mktemp + local LAST_ERR=/tmp/tmp.IDdG2OIQQL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5C4VyMwFR2 No resources found + cat /tmp/tmp.IDdG2OIQQL + rm /tmp/tmp.5C4VyMwFR2 /tmp/tmp.IDdG2OIQQL + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + awk '{print$1}' + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.L3tUA8Ogln ++ mktemp + local LAST_ERR=/tmp/tmp.Lmk1sN5QUr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns ++ mktemp + local LAST_OUT=/tmp/tmp.fsy5QeEUm3 + xargs kubectl delete ns + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_ERR=/tmp/tmp.Bx3A5rsL7G + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.L3tUA8Ogln + cat /tmp/tmp.Lmk1sN5QUr + rm /tmp/tmp.L3tUA8Ogln /tmp/tmp.Lmk1sN5QUr + return 0 namespace "scheduled-backup-26591" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fsy5QeEUm3 namespace "pxc-operator" deleted + cat /tmp/tmp.Bx3A5rsL7G + rm /tmp/tmp.fsy5QeEUm3 /tmp/tmp.Bx3A5rsL7G + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.7sLYArwOvr ++ mktemp + local LAST_ERR=/tmp/tmp.PpDvfxK3aE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7sLYArwOvr namespace/pxc-operator created + cat /tmp/tmp.PpDvfxK3aE + rm /tmp/tmp.7sLYArwOvr /tmp/tmp.PpDvfxK3aE + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.BSxn6tuzmf +++ mktemp ++ local LAST_ERR=/tmp/tmp.wF3B4zB2Cu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BSxn6tuzmf ++ cat /tmp/tmp.wF3B4zB2Cu ++ rm /tmp/tmp.BSxn6tuzmf /tmp/tmp.wF3B4zB2Cu ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster1 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ACtKDdgEMx ++ mktemp + local LAST_ERR=/tmp/tmp.5ZdkAtgDzf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster1 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ACtKDdgEMx Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster1" modified. + cat /tmp/tmp.5ZdkAtgDzf + rm /tmp/tmp.ACtKDdgEMx /tmp/tmp.5ZdkAtgDzf + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.hpDdLdUFTe ++ mktemp + local LAST_ERR=/tmp/tmp.2FeYmzKbhF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hpDdLdUFTe customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.2FeYmzKbhF + rm /tmp/tmp.hpDdLdUFTe /tmp/tmp.2FeYmzKbhF + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.PJeINKJTVd ++ mktemp + local LAST_ERR=/tmp/tmp.WWt7wqGrVJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PJeINKJTVd clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.WWt7wqGrVJ + rm /tmp/tmp.PJeINKJTVd /tmp/tmp.WWt7wqGrVJ + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/deploy/cw-operator.yaml + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + kubectl_bin apply -f - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6^' ++ mktemp + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + local LAST_OUT=/tmp/tmp.XTO4vwKU1J ++ mktemp + local LAST_ERR=/tmp/tmp.3L68RwdvVa + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XTO4vwKU1J deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.3L68RwdvVa + rm /tmp/tmp.XTO4vwKU1J /tmp/tmp.3L68RwdvVa + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.oHEZW9bNun ++ mktemp + local LAST_ERR=/tmp/tmp.MsAqUIFQDR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oHEZW9bNun pod/percona-xtradb-cluster-operator-75f64b98f6-4z44r condition met + cat /tmp/tmp.MsAqUIFQDR + rm /tmp/tmp.oHEZW9bNun /tmp/tmp.MsAqUIFQDR + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.iai8b3n2Ah +++ mktemp ++ local LAST_ERR=/tmp/tmp.N8U7GloErC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iai8b3n2Ah ++ cat /tmp/tmp.N8U7GloErC ++ rm /tmp/tmp.iai8b3n2Ah /tmp/tmp.N8U7GloErC ++ return 0 + wait_pod percona-xtradb-cluster-operator-75f64b98f6-4z44r 480 pxc-operator + local pod=percona-xtradb-cluster-operator-75f64b98f6-4z44r + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-75f64b98f6-4z44r ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-75f64b98f6-4z44r condition met waiting for pod/percona-xtradb-cluster-operator-75f64b98f6-4z44r to become Ready.Ok + sleep 3 + create_namespace scheduled-backup-25427 + local namespace=scheduled-backup-25427 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ awk '-F ' '{print $2}' ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrole ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces scheduled-backup-25427' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces scheduled-backup-25427 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace scheduled-backup-25427 ++ mktemp + local LAST_OUT=/tmp/tmp.KBY5clIXy7 + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.R00XEqAyhN + local exit_status=0 + local LAST_OUT=/tmp/tmp.PHAURT0Tzh ++ seq 0 2 ++ mktemp + local LAST_ERR=/tmp/tmp.ubUwHdV4ox + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + awk '{print$1}' + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-25427 + xargs kubectl delete ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-25427 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PHAURT0Tzh + cat /tmp/tmp.ubUwHdV4ox + rm /tmp/tmp.PHAURT0Tzh /tmp/tmp.ubUwHdV4ox + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-25427 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.KBY5clIXy7 + cat /tmp/tmp.R00XEqAyhN Error from server (NotFound): namespaces "scheduled-backup-25427" not found + rm /tmp/tmp.KBY5clIXy7 /tmp/tmp.R00XEqAyhN + return 1 + : + wait_for_delete namespace/scheduled-backup-25427 + local res=namespace/scheduled-backup-25427 + echo -n 'waiting for namespace/scheduled-backup-25427 to be deleted' waiting for namespace/scheduled-backup-25427 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "scheduled-backup-25427" not found + desc 'create namespace scheduled-backup-25427' + set +o xtrace ----------------------------------------------------------------------------------- create namespace scheduled-backup-25427 ----------------------------------------------------------------------------------- + kubectl_bin create namespace scheduled-backup-25427 ++ mktemp + local LAST_OUT=/tmp/tmp.Cj3YoqRSHc ++ mktemp + local LAST_ERR=/tmp/tmp.JjlST7npaV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace scheduled-backup-25427 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Cj3YoqRSHc namespace/scheduled-backup-25427 created + cat /tmp/tmp.JjlST7npaV + rm /tmp/tmp.Cj3YoqRSHc /tmp/tmp.JjlST7npaV + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.t2pW3QLgF8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.yv4uYOHKvj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.t2pW3QLgF8 ++ cat /tmp/tmp.yv4uYOHKvj ++ rm /tmp/tmp.t2pW3QLgF8 /tmp/tmp.yv4uYOHKvj ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster1 --namespace=scheduled-backup-25427 ++ mktemp + local LAST_OUT=/tmp/tmp.qGPLaRUaj8 ++ mktemp + local LAST_ERR=/tmp/tmp.1Ds4b24Qav + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster1 --namespace=scheduled-backup-25427 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qGPLaRUaj8 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster1" modified. + cat /tmp/tmp.1Ds4b24Qav + rm /tmp/tmp.qGPLaRUaj8 /tmp/tmp.1Ds4b24Qav + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.tjQcZt68tu ++ mktemp + local LAST_ERR=/tmp/tmp.qx2bvDoEcE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tjQcZt68tu secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.qx2bvDoEcE + rm /tmp/tmp.tjQcZt68tu /tmp/tmp.qx2bvDoEcE + return 0 + start_minio + deploy_helm scheduled-backup-25427 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "hashicorp" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Wed Nov 12 13:23:46 2025 NAMESPACE: scheduled-backup-25427 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.scheduled-backup-25427.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-25427 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-25427 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-25427 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-25427 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xArBZ7s4fB +++ mktemp ++ local LAST_ERR=/tmp/tmp.iF3QSUfc4K ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xArBZ7s4fB ++ cat /tmp/tmp.iF3QSUfc4K ++ rm /tmp/tmp.xArBZ7s4fB /tmp/tmp.iF3QSUfc4K ++ return 0 + MINIO_POD=minio-service-55fcc5d75f-7kpqc + wait_pod minio-service-55fcc5d75f-7kpqc + local pod=minio-service-55fcc5d75f-7kpqc + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo minio-service-55fcc5d75f-7kpqc + local container= + set +o xtrace pod/minio-service-55fcc5d75f-7kpqc condition met waiting for pod/minio-service-55fcc5d75f-7kpqc to become Ready.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.1r884dRq9S ++ mktemp + local LAST_ERR=/tmp/tmp.yffLayKOOW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1r884dRq9S make_bucket: operator-testing pod "aws-cli" deleted from scheduled-backup-25427 namespace + cat /tmp/tmp.yffLayKOOW All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_scheduled-backup-25427 + rm /tmp/tmp.1r884dRq9S /tmp/tmp.yffLayKOOW + return 0 + cluster=scheduled-backup + cat - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.69dLxJPKdo ++ mktemp + local LAST_ERR=/tmp/tmp.y8T9cyzGgj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.69dLxJPKdo priorityclass.scheduling.k8s.io/high-priority configured + cat /tmp/tmp.y8T9cyzGgj + rm /tmp/tmp.69dLxJPKdo /tmp/tmp.y8T9cyzGgj + return 0 + spinup_pxc scheduled-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local cluster=scheduled-backup + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.geOCISmqTC ++ mktemp + local LAST_ERR=/tmp/tmp.5nhMrtkpuT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.geOCISmqTC secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.5nhMrtkpuT + rm /tmp/tmp.geOCISmqTC /tmp/tmp.5nhMrtkpuT + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/client.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-25427~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + local LAST_OUT=/tmp/tmp.Tnde3mNhmG ++ mktemp + local LAST_ERR=/tmp/tmp.CcJgiwXlAy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Tnde3mNhmG deployment.apps/pxc-client created + cat /tmp/tmp.CcJgiwXlAy + rm /tmp/tmp.Tnde3mNhmG /tmp/tmp.CcJgiwXlAy + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_OUT=/tmp/tmp.7CqM1rtfat + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-25427~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_ERR=/tmp/tmp.UG3ObUPnIY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7CqM1rtfat perconaxtradbcluster.pxc.percona.com/scheduled-backup created + cat /tmp/tmp.UG3ObUPnIY + rm /tmp/tmp.7CqM1rtfat /tmp/tmp.UG3ObUPnIY + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy scheduled-backup ++ local target_cluster=scheduled-backup +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.665RQYLr5W ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TN64l7gfkL +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.665RQYLr5W +++ cat /tmp/tmp.TN64l7gfkL +++ rm /tmp/tmp.665RQYLr5W /tmp/tmp.TN64l7gfkL +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.YRU6Rs6qXT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.CCwP5yyNmw +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.YRU6Rs6qXT +++ cat /tmp/tmp.CCwP5yyNmw +++ rm /tmp/tmp.YRU6Rs6qXT /tmp/tmp.CCwP5yyNmw +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scheduled-backup-proxysql ++ return + local proxy=scheduled-backup-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-25427 ++ mktemp + local LAST_OUT=/tmp/tmp.di2itJevfF ++ mktemp + local LAST_ERR=/tmp/tmp.msV3QijqMK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-25427 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-25427 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-25427 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.di2itJevfF + cat /tmp/tmp.msV3QijqMK error: no matching resources found + rm /tmp/tmp.di2itJevfF /tmp/tmp.msV3QijqMK + return 1 + true + wait_for_running scheduled-backup-proxysql 1 + local name=scheduled-backup-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-proxysql-0 480 + local pod=scheduled-backup-proxysql-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/scheduled-backup-proxysql-0 condition met waiting for pod/scheduled-backup-proxysql-0 to become Ready.Ok + wait_for_running scheduled-backup-pxc 3 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 480 + local pod=scheduled-backup-pxc-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 480 + local pod=scheduled-backup-pxc-1 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 480 + local pod=scheduled-backup-pxc-2 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-2 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.QuzNw9AYHy +++ mktemp ++ local LAST_ERR=/tmp/tmp.Q2i036n8Cp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QuzNw9AYHy ++ cat /tmp/tmp.Q2i036n8Cp ++ rm /tmp/tmp.QuzNw9AYHy /tmp/tmp.Q2i036n8Cp ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nrtVogUlvu +++ mktemp ++ local LAST_ERR=/tmp/tmp.ilxqIlOOb5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nrtVogUlvu ++ cat /tmp/tmp.ilxqIlOOb5 ++ rm /tmp/tmp.nrtVogUlvu /tmp/tmp.ilxqIlOOb5 ++ return 0 + client_pod=pxc-client-59944c5bbf-qhhvg + wait_pod pxc-client-59944c5bbf-qhhvg + local pod=pxc-client-59944c5bbf-qhhvg + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-59944c5bbf-qhhvg + local container= + set +o xtrace pod/pxc-client-59944c5bbf-qhhvg condition met waiting for pod/pxc-client-59944c5bbf-qhhvg to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9EqMS6pnrt +++ mktemp ++ local LAST_ERR=/tmp/tmp.hFHlxdwsWX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9EqMS6pnrt ++ cat /tmp/tmp.hFHlxdwsWX ++ rm /tmp/tmp.9EqMS6pnrt /tmp/tmp.hFHlxdwsWX ++ return 0 + client_pod=pxc-client-59944c5bbf-qhhvg + wait_pod pxc-client-59944c5bbf-qhhvg + local pod=pxc-client-59944c5bbf-qhhvg + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-qhhvg ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-qhhvg condition met waiting for pod/pxc-client-59944c5bbf-qhhvg to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JRmR9s8obj +++ mktemp ++ local LAST_ERR=/tmp/tmp.kg7CkN1KvS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JRmR9s8obj ++ cat /tmp/tmp.kg7CkN1KvS ++ rm /tmp/tmp.JRmR9s8obj /tmp/tmp.kg7CkN1KvS ++ return 0 + client_pod=pxc-client-59944c5bbf-qhhvg + wait_pod pxc-client-59944c5bbf-qhhvg + local pod=pxc-client-59944c5bbf-qhhvg + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-qhhvg ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-qhhvg condition met waiting for pod/pxc-client-59944c5bbf-qhhvg to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.KFnSFqE6Mr/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.KFnSFqE6Mr/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DFeO7SaH16 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FULpErTuXl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DFeO7SaH16 ++ cat /tmp/tmp.FULpErTuXl ++ rm /tmp/tmp.DFeO7SaH16 /tmp/tmp.FULpErTuXl ++ return 0 + client_pod=pxc-client-59944c5bbf-qhhvg + wait_pod pxc-client-59944c5bbf-qhhvg + local pod=pxc-client-59944c5bbf-qhhvg + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-qhhvg ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-qhhvg condition met waiting for pod/pxc-client-59944c5bbf-qhhvg to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.KFnSFqE6Mr/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.KFnSFqE6Mr/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MABRCW8zhu +++ mktemp ++ local LAST_ERR=/tmp/tmp.ngIojZ9zM4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MABRCW8zhu ++ cat /tmp/tmp.ngIojZ9zM4 ++ rm /tmp/tmp.MABRCW8zhu /tmp/tmp.ngIojZ9zM4 ++ return 0 + client_pod=pxc-client-59944c5bbf-qhhvg + wait_pod pxc-client-59944c5bbf-qhhvg + local pod=pxc-client-59944c5bbf-qhhvg + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-59944c5bbf-qhhvg ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-qhhvg condition met waiting for pod/pxc-client-59944c5bbf-qhhvg to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.KFnSFqE6Mr/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.KFnSFqE6Mr/select-1.sql ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2NB2Q3lGUV +++ mktemp ++ local LAST_ERR=/tmp/tmp.VCM0mRYYRI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2NB2Q3lGUV ++ cat /tmp/tmp.VCM0mRYYRI Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.2NB2Q3lGUV /tmp/tmp.VCM0mRYYRI ++ return 0 + '[' '' ']' + sleep 20 + desc 'add backups schedule for pvc storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for pvc storage ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.7WkkrL5x9d +++ mktemp ++ local LAST_ERR=/tmp/tmp.T8w0O9YVAX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7WkkrL5x9d ++ cat /tmp/tmp.T8w0O9YVAX ++ rm /tmp/tmp.7WkkrL5x9d /tmp/tmp.T8w0O9YVAX ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster1 --namespace=scheduled-backup-25427 ++ mktemp + local LAST_OUT=/tmp/tmp.EvRIDG0Ln6 ++ mktemp + local LAST_ERR=/tmp/tmp.btfMVr4KWh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster1 --namespace=scheduled-backup-25427 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EvRIDG0Ln6 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2235-13a5e3e6-1-cluster1" modified. + cat /tmp/tmp.btfMVr4KWh + rm /tmp/tmp.EvRIDG0Ln6 /tmp/tmp.btfMVr4KWh + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.th0OmI2O6l + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-25427~ + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.9YloIyTtQa + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.th0OmI2O6l perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.9YloIyTtQa + rm /tmp/tmp.th0OmI2O6l /tmp/tmp.9YloIyTtQa + return 0 + label_node ++ kubectl_bin get nodes --no-headers=true ++ grep -v master ++ head -n1 +++ mktemp ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.eq7Y2lWgZc +++ mktemp ++ local LAST_ERR=/tmp/tmp.PtRiBMyUIl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get nodes --no-headers=true ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eq7Y2lWgZc ++ cat /tmp/tmp.PtRiBMyUIl ++ rm /tmp/tmp.eq7Y2lWgZc /tmp/tmp.PtRiBMyUIl ++ return 0 + LABELED_NODE=gke-jen-pxc-2235-13a5e3e-default-pool-b50c7585-35r8 + kubectl_bin label nodes gke-jen-pxc-2235-13a5e3e-default-pool-b50c7585-35r8 backupWorker=True --overwrite ++ mktemp + local LAST_OUT=/tmp/tmp.ZZoEe5W04u ++ mktemp + local LAST_ERR=/tmp/tmp.7NOOppZI4V + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label nodes gke-jen-pxc-2235-13a5e3e-default-pool-b50c7585-35r8 backupWorker=True --overwrite + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZZoEe5W04u node/gke-jen-pxc-2235-13a5e3e-default-pool-b50c7585-35r8 not labeled + cat /tmp/tmp.7NOOppZI4V + rm /tmp/tmp.ZZoEe5W04u /tmp/tmp.7NOOppZI4V + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-25427~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_OUT=/tmp/tmp.z5jnhC9vb9 ++ mktemp + local LAST_ERR=/tmp/tmp.tYdqpEcjwS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.z5jnhC9vb9 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.tYdqpEcjwS + rm /tmp/tmp.z5jnhC9vb9 /tmp/tmp.tYdqpEcjwS + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.GSn3t70ulb +++ mktemp ++ local LAST_ERR=/tmp/tmp.BcQjvhUwXH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GSn3t70ulb ++ cat /tmp/tmp.BcQjvhUwXH ++ rm /tmp/tmp.GSn3t70ulb /tmp/tmp.BcQjvhUwXH ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.pakhBvzqY7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.YpfZPtwCog ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pakhBvzqY7 ++ cat /tmp/tmp.YpfZPtwCog ++ rm /tmp/tmp.pakhBvzqY7 /tmp/tmp.YpfZPtwCog ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-0 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.WAR20YLEik ++ mktemp + local LAST_ERR=/tmp/tmp.o0hCDYcdKe + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WAR20YLEik NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-20251112133120-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112133120-q6fav Running 27s + cat /tmp/tmp.o0hCDYcdKe + rm /tmp/tmp.WAR20YLEik /tmp/tmp.o0hCDYcdKe + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.1jAdw7Gj7r ++ mktemp + local LAST_ERR=/tmp/tmp.CxFPI94WK4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1jAdw7Gj7r NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-7kpqc 1/1 Running 0 7m59s pxc-client-59944c5bbf-qhhvg 2/2 Running 0 6m49s scheduled-backup-proxysql-0 3/3 Running 0 6m47s scheduled-backup-proxysql-1 3/3 Running 0 6m31s scheduled-backup-pxc-0 1/1 Running 0 6m47s scheduled-backup-pxc-1 1/1 Running 0 5m34s scheduled-backup-pxc-2 1/1 Running 0 4m21s xb-cron-scheduled-backup-pvc-20251112133120-q6fav-bccz4 1/1 Running 0 28s + cat /tmp/tmp.CxFPI94WK4 + rm /tmp/tmp.1jAdw7Gj7r /tmp/tmp.CxFPI94WK4 + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.EV6J85iBrV +++ mktemp ++ local LAST_ERR=/tmp/tmp.IguE1UGMln ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EV6J85iBrV ++ cat /tmp/tmp.IguE1UGMln ++ rm /tmp/tmp.EV6J85iBrV /tmp/tmp.IguE1UGMln ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.2asHawaeNv +++ mktemp ++ local LAST_ERR=/tmp/tmp.kixgaT4Rg0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2asHawaeNv ++ cat /tmp/tmp.kixgaT4Rg0 ++ rm /tmp/tmp.2asHawaeNv /tmp/tmp.kixgaT4Rg0 ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-25427~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + local LAST_OUT=/tmp/tmp.6le9NoUoCB + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_ERR=/tmp/tmp.6HvCQVnU9o + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6le9NoUoCB perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.6HvCQVnU9o + rm /tmp/tmp.6le9NoUoCB /tmp/tmp.6HvCQVnU9o + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-25427~ ++ mktemp + local LAST_OUT=/tmp/tmp.8c6iruNw5E ++ mktemp + local LAST_ERR=/tmp/tmp.dY2mZfuBUd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8c6iruNw5E perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.dY2mZfuBUd + rm /tmp/tmp.8c6iruNw5E /tmp/tmp.dY2mZfuBUd + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.2AHqSy0sDU +++ mktemp ++ local LAST_ERR=/tmp/tmp.wsnAD0xTEy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2AHqSy0sDU ++ cat /tmp/tmp.wsnAD0xTEy ++ rm /tmp/tmp.2AHqSy0sDU /tmp/tmp.wsnAD0xTEy ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.HliOz3ST03 +++ mktemp ++ local LAST_ERR=/tmp/tmp.hcvSsjjxIC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HliOz3ST03 ++ cat /tmp/tmp.hcvSsjjxIC ++ rm /tmp/tmp.HliOz3ST03 /tmp/tmp.hcvSsjjxIC ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.qGh0RvAIyn ++ mktemp + local LAST_ERR=/tmp/tmp.AzyvRJh07k + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qGh0RvAIyn NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112133313-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-11-12-13:33:13-full Running 22s cron-scheduled-backup-pvc-20251112133120-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112133120-q6fav Succeeded 99s 2m15s + cat /tmp/tmp.AzyvRJh07k + rm /tmp/tmp.qGh0RvAIyn /tmp/tmp.AzyvRJh07k + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.0rDxkKjjri ++ mktemp + local LAST_ERR=/tmp/tmp.1LsedCcDqI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0rDxkKjjri NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-7kpqc 1/1 Running 0 9m47s pxc-client-59944c5bbf-qhhvg 2/2 Running 0 8m37s scheduled-backup-proxysql-0 3/3 Running 0 8m35s scheduled-backup-proxysql-1 3/3 Running 0 8m19s scheduled-backup-pxc-0 1/1 Running 0 8m35s scheduled-backup-pxc-1 1/1 Running 0 7m22s scheduled-backup-pxc-2 1/1 Running 0 6m9s xb-cron-scheduled-backup-aws-s3-20251112133313-q6fav-mtdt6 1/1 Running 0 23s xb-cron-scheduled-backup-pvc-20251112133120-q6fav-bccz4 0/1 Completed 0 2m16s + cat /tmp/tmp.1LsedCcDqI + rm /tmp/tmp.0rDxkKjjri /tmp/tmp.1LsedCcDqI + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.GXwd3LiO1y +++ mktemp ++ local LAST_ERR=/tmp/tmp.c0eSdWdR5r ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GXwd3LiO1y ++ cat /tmp/tmp.c0eSdWdR5r ++ rm /tmp/tmp.GXwd3LiO1y /tmp/tmp.c0eSdWdR5r ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.QQa7xkVyQJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.slwHwrkotw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QQa7xkVyQJ ++ cat /tmp/tmp.slwHwrkotw ++ rm /tmp/tmp.QQa7xkVyQJ /tmp/tmp.slwHwrkotw ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for minio storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for minio storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-25427~ + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + local LAST_OUT=/tmp/tmp.lsqMQf7AzN ++ mktemp + local LAST_ERR=/tmp/tmp.AOChsFpGXD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lsqMQf7AzN perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.AOChsFpGXD + rm /tmp/tmp.lsqMQf7AzN /tmp/tmp.AOChsFpGXD + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-25427~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.2QdyIh1vq4 ++ mktemp + local LAST_ERR=/tmp/tmp.LBKtoLzEJ2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2QdyIh1vq4 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.LBKtoLzEJ2 + rm /tmp/tmp.2QdyIh1vq4 /tmp/tmp.LBKtoLzEJ2 + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.rDnfHXcdsK +++ mktemp ++ local LAST_ERR=/tmp/tmp.UV6VV9JQ2G ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rDnfHXcdsK ++ cat /tmp/tmp.UV6VV9JQ2G ++ rm /tmp/tmp.rDnfHXcdsK /tmp/tmp.UV6VV9JQ2G ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ grep :Failed ++ local LAST_OUT=/tmp/tmp.ZI4ou6hiCD ++ wc -l +++ mktemp ++ local LAST_ERR=/tmp/tmp.zlRAk6FK9G ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZI4ou6hiCD ++ cat /tmp/tmp.zlRAk6FK9G ++ rm /tmp/tmp.ZI4ou6hiCD /tmp/tmp.zlRAk6FK9G ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-2 ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.6Tfj4T9x4C ++ mktemp + local LAST_ERR=/tmp/tmp.AZMAbCdJPx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6Tfj4T9x4C NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112133313-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-11-12-13:33:13-full Succeeded 101s 2m14s cron-scheduled-backup-minio-2025111213350-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-11-12-13:35:00-full Running 27s cron-scheduled-backup-pvc-20251112133120-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112133120-q6fav Succeeded 3m31s 4m7s + cat /tmp/tmp.AZMAbCdJPx + rm /tmp/tmp.6Tfj4T9x4C /tmp/tmp.AZMAbCdJPx + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.887ZxcIM2X ++ mktemp + local LAST_ERR=/tmp/tmp.bwXC7ZEyl6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.887ZxcIM2X NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-7kpqc 1/1 Running 0 11m pxc-client-59944c5bbf-qhhvg 2/2 Running 0 10m scheduled-backup-proxysql-0 3/3 Running 0 10m scheduled-backup-proxysql-1 3/3 Running 0 10m scheduled-backup-pxc-0 1/1 Running 0 10m scheduled-backup-pxc-1 1/1 Running 0 9m14s scheduled-backup-pxc-2 1/1 Running 0 8m1s xb-cron-scheduled-backup-aws-s3-20251112133313-q6fav-mtdt6 0/1 Completed 0 2m15s xb-cron-scheduled-backup-minio-2025111213350-q6fav-pxxsr 0/1 Completed 0 28s xb-cron-scheduled-backup-pvc-20251112133120-q6fav-bccz4 0/1 Completed 0 4m8s + cat /tmp/tmp.bwXC7ZEyl6 + rm /tmp/tmp.887ZxcIM2X /tmp/tmp.bwXC7ZEyl6 + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZfsoCKpNSf +++ mktemp ++ local LAST_ERR=/tmp/tmp.WN2qwjBdKi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZfsoCKpNSf ++ cat /tmp/tmp.WN2qwjBdKi ++ rm /tmp/tmp.ZfsoCKpNSf /tmp/tmp.WN2qwjBdKi ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ grep :Failed ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.cRm77Ri9GW +++ mktemp ++ local LAST_ERR=/tmp/tmp.10cYb2xJ1K ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cRm77Ri9GW ++ cat /tmp/tmp.10cYb2xJ1K ++ rm /tmp/tmp.cRm77Ri9GW /tmp/tmp.10cYb2xJ1K ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for gcs storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for gcs storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-25427~ + local LAST_OUT=/tmp/tmp.YbOZCauhVe + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_ERR=/tmp/tmp.9ZbNGX4MQW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YbOZCauhVe perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.9ZbNGX4MQW + rm /tmp/tmp.YbOZCauhVe /tmp/tmp.9ZbNGX4MQW + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-25427~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.1BdS14Jrya ++ mktemp + local LAST_ERR=/tmp/tmp.Wk88K5mDP8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1BdS14Jrya perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.Wk88K5mDP8 + rm /tmp/tmp.1BdS14Jrya /tmp/tmp.Wk88K5mDP8 + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ie5lCCegF5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.d5bpazQZbn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ie5lCCegF5 ++ cat /tmp/tmp.d5bpazQZbn ++ rm /tmp/tmp.Ie5lCCegF5 /tmp/tmp.d5bpazQZbn ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ grep :Failed ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3xbO4TrMqH +++ mktemp ++ local LAST_ERR=/tmp/tmp.Zs3nVJ1Bha ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3xbO4TrMqH ++ cat /tmp/tmp.Zs3nVJ1Bha ++ rm /tmp/tmp.3xbO4TrMqH /tmp/tmp.Zs3nVJ1Bha ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.JkSgNs40nk ++ mktemp + local LAST_ERR=/tmp/tmp.scv3nfn6gT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JkSgNs40nk NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112133313-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-11-12-13:33:13-full Succeeded 3m31s 4m4s cron-scheduled-backup-gcp-cs-20251112133654-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2025-11-12-13:36:54-full Running 23s cron-scheduled-backup-minio-2025111213350-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-11-12-13:35:00-full Succeeded 109s 2m17s cron-scheduled-backup-pvc-20251112133120-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112133120-q6fav Succeeded 5m21s 5m57s + cat /tmp/tmp.scv3nfn6gT + rm /tmp/tmp.JkSgNs40nk /tmp/tmp.scv3nfn6gT + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.DdDFTePBrS ++ mktemp + local LAST_ERR=/tmp/tmp.U9uiP0auYv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DdDFTePBrS NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-7kpqc 1/1 Running 0 13m pxc-client-59944c5bbf-qhhvg 2/2 Running 0 12m scheduled-backup-proxysql-0 3/3 Running 0 12m scheduled-backup-proxysql-1 3/3 Running 0 12m scheduled-backup-pxc-0 1/1 Running 0 12m scheduled-backup-pxc-1 1/1 Running 0 11m scheduled-backup-pxc-2 1/1 Running 0 9m51s xb-cron-scheduled-backup-aws-s3-20251112133313-q6fav-mtdt6 0/1 Completed 0 4m5s xb-cron-scheduled-backup-gcp-cs-20251112133654-q6fav-pvl98 1/1 Running 0 24s xb-cron-scheduled-backup-minio-2025111213350-q6fav-pxxsr 0/1 Completed 0 2m18s xb-cron-scheduled-backup-pvc-20251112133120-q6fav-bccz4 0/1 Completed 0 5m58s + cat /tmp/tmp.U9uiP0auYv + rm /tmp/tmp.DdDFTePBrS /tmp/tmp.U9uiP0auYv + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SbPKg2ep79 +++ mktemp ++ local LAST_ERR=/tmp/tmp.99e4rKiZan ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SbPKg2ep79 ++ cat /tmp/tmp.99e4rKiZan ++ rm /tmp/tmp.SbPKg2ep79 /tmp/tmp.99e4rKiZan ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ wc -l ++ grep :Failed ++ local LAST_OUT=/tmp/tmp.7WMEu0WFc7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.OLdCPo9QtB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7WMEu0WFc7 ++ cat /tmp/tmp.OLdCPo9QtB ++ rm /tmp/tmp.7WMEu0WFc7 /tmp/tmp.OLdCPo9QtB ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for azure storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for azure storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-25427~ ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_OUT=/tmp/tmp.ydZ0CXwqwT + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ mktemp + local LAST_ERR=/tmp/tmp.qmqMqsTZyz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ydZ0CXwqwT perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.qmqMqsTZyz + rm /tmp/tmp.ydZ0CXwqwT /tmp/tmp.qmqMqsTZyz + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2235/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2235-13a5e3e6#' + local LAST_OUT=/tmp/tmp.cj957FpXf0 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + local LAST_ERR=/tmp/tmp.EUYzNiYosp + local exit_status=0 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ seq 0 2 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-25427~ + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cj957FpXf0 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.EUYzNiYosp + rm /tmp/tmp.cj957FpXf0 /tmp/tmp.EUYzNiYosp + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Wn7fRBD5KB +++ mktemp ++ local LAST_ERR=/tmp/tmp.g30E7BtyTa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Wn7fRBD5KB ++ cat /tmp/tmp.g30E7BtyTa ++ rm /tmp/tmp.Wn7fRBD5KB /tmp/tmp.g30E7BtyTa ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.LGcgPqE88x +++ mktemp ++ local LAST_ERR=/tmp/tmp.9YMqSK5ujK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LGcgPqE88x ++ cat /tmp/tmp.9YMqSK5ujK ++ rm /tmp/tmp.LGcgPqE88x /tmp/tmp.9YMqSK5ujK ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.8q1tXJzqEx ++ mktemp + local LAST_ERR=/tmp/tmp.KCLGSKFbpO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8q1tXJzqEx NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251112133313-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-11-12-13:33:13-full Succeeded 5m14s 5m47s cron-scheduled-backup-azure-blob-20251112133845-q6fav scheduled-backup azure-blob azure://operator-testing/scheduled-backup-2025-11-12-13:38:45-full Running 15s cron-scheduled-backup-gcp-cs-20251112133654-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2025-11-12-13:36:54-full Succeeded 95s 2m6s cron-scheduled-backup-minio-2025111213350-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-11-12-13:35:00-full Succeeded 3m32s 4m cron-scheduled-backup-pvc-20251112133120-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251112133120-q6fav Succeeded 7m4s 7m40s + cat /tmp/tmp.KCLGSKFbpO + rm /tmp/tmp.8q1tXJzqEx /tmp/tmp.KCLGSKFbpO + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.58K043hEBB ++ mktemp + local LAST_ERR=/tmp/tmp.z9A3gMLmkl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.58K043hEBB NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-7kpqc 1/1 Running 0 15m pxc-client-59944c5bbf-qhhvg 2/2 Running 0 14m scheduled-backup-proxysql-0 3/3 Running 0 13m scheduled-backup-proxysql-1 3/3 Running 0 13m scheduled-backup-pxc-0 1/1 Running 0 13m scheduled-backup-pxc-1 1/1 Running 0 12m scheduled-backup-pxc-2 1/1 Running 0 11m xb-cron-scheduled-backup-aws-s3-20251112133313-q6fav-mtdt6 0/1 Completed 0 5m47s xb-cron-scheduled-backup-azure-blob-20251112133845-q6fav-shxjj 1/1 Running 0 15s xb-cron-scheduled-backup-gcp-cs-20251112133654-q6fav-pvl98 0/1 Completed 0 2m6s xb-cron-scheduled-backup-minio-2025111213350-q6fav-pxxsr 0/1 Completed 0 4m xb-cron-scheduled-backup-pvc-20251112133120-q6fav-bccz4 0/1 Completed 0 7m40s + cat /tmp/tmp.z9A3gMLmkl + rm /tmp/tmp.58K043hEBB /tmp/tmp.z9A3gMLmkl + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3pH3gIOOYK +++ mktemp ++ local LAST_ERR=/tmp/tmp.LYQe1lGJGy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3pH3gIOOYK ++ cat /tmp/tmp.LYQe1lGJGy ++ rm /tmp/tmp.3pH3gIOOYK /tmp/tmp.LYQe1lGJGy ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ wc -l ++ grep :Failed ++ local LAST_OUT=/tmp/tmp.OBr4iSuLck +++ mktemp ++ local LAST_ERR=/tmp/tmp.dphTXqas7B ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OBr4iSuLck ++ cat /tmp/tmp.dphTXqas7B ++ rm /tmp/tmp.OBr4iSuLck /tmp/tmp.dphTXqas7B ++ return 0 + [[ 0 -gt 0 ]] ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ head -n1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.taGGYEI0QI +++ mktemp ++ local LAST_ERR=/tmp/tmp.AOqzqnawUH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ cut -d: -f1 ++ grep pvc ++ grep Succeeded ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.taGGYEI0QI ++ cat /tmp/tmp.AOqzqnawUH ++ rm /tmp/tmp.taGGYEI0QI /tmp/tmp.AOqzqnawUH ++ return 0 + FIRST_PVC_BACKUP=cron-scheduled-backup-pvc-20251112133120-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-pvc-20251112133120-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.B284Dx0a4N +++ mktemp ++ local LAST_ERR=/tmp/tmp.EhwghXLs5r ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.B284Dx0a4N ++ cat /tmp/tmp.EhwghXLs5r ++ rm /tmp/tmp.B284Dx0a4N /tmp/tmp.EhwghXLs5r ++ return 0 + JOB_PVC_BACKUP=xb-cron-scheduled-backup-pvc-20251112133120-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-pvc-20251112133120 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HLvL0oye1z +++ mktemp ++ local LAST_ERR=/tmp/tmp.WgGKtvr6EN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HLvL0oye1z ++ cat /tmp/tmp.WgGKtvr6EN ++ rm /tmp/tmp.HLvL0oye1z /tmp/tmp.WgGKtvr6EN ++ return 0 + POD_PVC_BACKUP=xb-cron-scheduled-backup-pvc-20251112133120-q6fav-bccz4 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ grep minio ++ head -n1 ++ cut -d: -f1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.wggF6KlGln +++ mktemp ++ local LAST_ERR=/tmp/tmp.uULhUhwee8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wggF6KlGln ++ cat /tmp/tmp.uULhUhwee8 ++ rm /tmp/tmp.wggF6KlGln /tmp/tmp.uULhUhwee8 ++ return 0 + FIRST_MINIO_BACKUP=cron-scheduled-backup-minio-2025111213350-q6fav ++ kubectl_bin get jobs +++ mktemp ++ grep cron-scheduled-backup-minio-2025111213350-q6fav ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.ojDNhYt5Xj +++ mktemp ++ local LAST_ERR=/tmp/tmp.7cvpLFZhLA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ojDNhYt5Xj ++ cat /tmp/tmp.7cvpLFZhLA ++ rm /tmp/tmp.ojDNhYt5Xj /tmp/tmp.7cvpLFZhLA ++ return 0 + JOB_MINIO_BACKUP=xb-cron-scheduled-backup-minio-2025111213350-q6fav ++ kubectl_bin get pods +++ mktemp ++ grep xb-cron-scheduled-backup-minio-2025111213350 ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.BHqenjlLog +++ mktemp ++ local LAST_ERR=/tmp/tmp.UZX1v81izg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BHqenjlLog ++ cat /tmp/tmp.UZX1v81izg ++ rm /tmp/tmp.BHqenjlLog /tmp/tmp.UZX1v81izg ++ return 0 + POD_MINIO_BACKUP=xb-cron-scheduled-backup-minio-2025111213350-q6fav-pxxsr + '[' -z '' ']' ++ head -n1 ++ grep aws ++ cut -d: -f1 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded +++ mktemp ++ local LAST_OUT=/tmp/tmp.i3VwIzmjtP +++ mktemp ++ local LAST_ERR=/tmp/tmp.zG5WiIMv6b ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.i3VwIzmjtP ++ cat /tmp/tmp.zG5WiIMv6b ++ rm /tmp/tmp.i3VwIzmjtP /tmp/tmp.zG5WiIMv6b ++ return 0 + FIRST_AWS_BACKUP=cron-scheduled-backup-aws-s3-20251112133313-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-aws-s3-20251112133313-q6fav +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z0c6W5CMy2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gR60awSSk6 ++ local exit_status=0 ++ awk '{print $1}' +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Z0c6W5CMy2 ++ cat /tmp/tmp.gR60awSSk6 ++ rm /tmp/tmp.Z0c6W5CMy2 /tmp/tmp.gR60awSSk6 ++ return 0 + JOB_AWS_BACKUP=xb-cron-scheduled-backup-aws-s3-20251112133313-q6fav ++ grep xb-cron-scheduled-backup-aws-s3-20251112133313 ++ kubectl_bin get pods +++ mktemp ++ local LAST_OUT=/tmp/tmp.niGbhf9Y1Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.qd15mLWzaF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ awk '{print $1}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.niGbhf9Y1Q ++ cat /tmp/tmp.qd15mLWzaF ++ rm /tmp/tmp.niGbhf9Y1Q /tmp/tmp.qd15mLWzaF ++ return 0 + POD_AWS_BACKUP=xb-cron-scheduled-backup-aws-s3-20251112133313-q6fav-mtdt6 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded +++ mktemp ++ head -n1 ++ cut -d: -f1 ++ grep gcp ++ local LAST_OUT=/tmp/tmp.Mw7IEx612q +++ mktemp ++ local LAST_ERR=/tmp/tmp.HHLIWCMymv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Mw7IEx612q ++ cat /tmp/tmp.HHLIWCMymv ++ rm /tmp/tmp.Mw7IEx612q /tmp/tmp.HHLIWCMymv ++ return 0 + FIRST_GCP_BACKUP=cron-scheduled-backup-gcp-cs-20251112133654-q6fav ++ kubectl_bin get jobs ++ grep cron-scheduled-backup-gcp-cs-20251112133654-q6fav ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HkO1VizP1G +++ mktemp ++ local LAST_ERR=/tmp/tmp.rvjQADQKcW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HkO1VizP1G ++ cat /tmp/tmp.rvjQADQKcW ++ rm /tmp/tmp.HkO1VizP1G /tmp/tmp.rvjQADQKcW ++ return 0 + JOB_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-20251112133654-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-gcp-cs-20251112133654 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v8uDkG5D6O +++ mktemp ++ local LAST_ERR=/tmp/tmp.EIKkJpp3aO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.v8uDkG5D6O ++ cat /tmp/tmp.EIKkJpp3aO ++ rm /tmp/tmp.v8uDkG5D6O /tmp/tmp.EIKkJpp3aO ++ return 0 + POD_GCP_BACKUP=xb-cron-scheduled-backup-gcp-cs-20251112133654-q6fav-pvl98 ++ kubectl_bin get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ grep Succeeded ++ cut -d: -f1 ++ head -n1 ++ grep azure +++ mktemp ++ local LAST_OUT=/tmp/tmp.crz7rrXaoY +++ mktemp ++ local LAST_ERR=/tmp/tmp.XgCA8CjjkW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup -o 'jsonpath={range .items[*]}{.metadata.name}:{.spec.storageName}:{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.crz7rrXaoY ++ cat /tmp/tmp.XgCA8CjjkW ++ rm /tmp/tmp.crz7rrXaoY /tmp/tmp.XgCA8CjjkW ++ return 0 + FIRST_AZURE_BACKUP=cron-scheduled-backup-azure-blob-20251112133845-q6fav ++ kubectl_bin get jobs ++ awk '{print $1}' ++ grep cron-scheduled-backup-azure-blob-20251112133845-q6fav +++ mktemp ++ local LAST_OUT=/tmp/tmp.5nXFrsHTsS +++ mktemp ++ local LAST_ERR=/tmp/tmp.W3v4AOHDMy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get jobs ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5nXFrsHTsS ++ cat /tmp/tmp.W3v4AOHDMy ++ rm /tmp/tmp.5nXFrsHTsS /tmp/tmp.W3v4AOHDMy ++ return 0 + JOB_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-20251112133845-q6fav ++ kubectl_bin get pods ++ grep xb-cron-scheduled-backup-azure-blob-20251112133845 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VPXeJZMJxe +++ mktemp ++ local LAST_ERR=/tmp/tmp.rsiESb0jP6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VPXeJZMJxe ++ cat /tmp/tmp.rsiESb0jP6 ++ rm /tmp/tmp.VPXeJZMJxe /tmp/tmp.rsiESb0jP6 ++ return 0 + POD_AZURE_BACKUP=xb-cron-scheduled-backup-azure-blob-20251112133845-q6fav-shxjj ++ kubectl_bin get pxc-backup cron-scheduled-backup-aws-s3-20251112133313-q6fav -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ cut -c 6- +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ln7wSpuzus +++ mktemp ++ local LAST_ERR=/tmp/tmp.Io1F7alj7S ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-aws-s3-20251112133313-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ln7wSpuzus ++ cat /tmp/tmp.Io1F7alj7S ++ rm /tmp/tmp.Ln7wSpuzus /tmp/tmp.Io1F7alj7S ++ return 0 + BACKUP_DEST_AWS=operator-testing/scheduled-backup-2025-11-12-13:33:13-full ++ kubectl_bin get pxc-backup cron-scheduled-backup-gcp-cs-20251112133654-q6fav -o 'jsonpath={.status.destination}' ++ cut -c 6- ++ sed -e 's/.json$//' +++ mktemp ++ local LAST_OUT=/tmp/tmp.u3NDcyNsPX +++ mktemp ++ local LAST_ERR=/tmp/tmp.qCrhKQuMn7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-gcp-cs-20251112133654-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u3NDcyNsPX ++ cat /tmp/tmp.qCrhKQuMn7 ++ rm /tmp/tmp.u3NDcyNsPX /tmp/tmp.qCrhKQuMn7 ++ return 0 + BACKUP_DEST_GCP=operator-testing/scheduled-backup-2025-11-12-13:36:54-full ++ kubectl_bin get pxc-backup cron-scheduled-backup-azure-blob-20251112133845-q6fav -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ cut -c 9- +++ mktemp ++ local LAST_OUT=/tmp/tmp.bWBE3E3lEO +++ mktemp ++ local LAST_ERR=/tmp/tmp.S4yXDKZVHs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup cron-scheduled-backup-azure-blob-20251112133845-q6fav -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bWBE3E3lEO ++ cat /tmp/tmp.S4yXDKZVHs ++ rm /tmp/tmp.bWBE3E3lEO /tmp/tmp.S4yXDKZVHs ++ return 0 + BACKUP_DEST_AZURE=operator-testing/scheduled-backup-2025-11-12-13:38:45-full + desc 'Check backup existence' + set +o xtrace ----------------------------------------------------------------------------------- Check backup existence ----------------------------------------------------------------------------------- + check_backup_existence https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 aws-s3 + path=https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + storage_name=aws-s3 + retry=0 ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 0 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 1 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 2 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 3 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 4 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 5 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 6 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 7 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 8 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 9 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 10 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 11 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 12 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 13 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 14 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 15 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 16 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 17 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 18 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 19 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 20 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 21 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 22 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 23 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 24 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 25 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 26 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 27 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 28 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 29 -ge 30 ']' + echo 'waiting for backup in aws-s3' waiting for backup in aws-s3 + sleep 10 + (( retry += 1 )) ++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/scheduled-backup-2025-11-12-13:33:13-full.sst_info/sst_info.00000000000000000000 + [[ 403 -eq 200 ]] + '[' 30 -ge 30 ']' + echo 'max retry count 30 reached. something went wrong with operator or kubernetes cluster' max retry count 30 reached. something went wrong with operator or kubernetes cluster + echo 'Backup was not found in bucket -- aws-s3' Backup was not found in bucket -- aws-s3 + exit 1