Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/logs/scheduled-backup-5-7.log Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 + main + create_infra scheduled-backup-15195 + local ns=scheduled-backup-15195 + '[' -n pxc-operator ']' + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n scheduled-backup-12053 scheduled-backup --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.5R85NjCUeq ++ mktemp + local LAST_ERR=/tmp/tmp.PBqelQ7HQD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5R85NjCUeq perconaxtradbcluster.pxc.percona.com "scheduled-backup" deleted from scheduled-backup-12053 namespace + cat /tmp/tmp.PBqelQ7HQD + rm /tmp/tmp.5R85NjCUeq /tmp/tmp.PBqelQ7HQD + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.2PEHAM0svc ++ mktemp + local LAST_ERR=/tmp/tmp.hmmeenQprm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2PEHAM0svc perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-aws-s3-202512412340-q6fav" deleted from scheduled-backup-12053 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-pvc-2025124123210-q6fav" deleted from scheduled-backup-12053 namespace + cat /tmp/tmp.hmmeenQprm + rm /tmp/tmp.2PEHAM0svc /tmp/tmp.hmmeenQprm + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.1DUmmEYBXK ++ mktemp + local LAST_ERR=/tmp/tmp.lQKWpdmZ4M + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1DUmmEYBXK No resources found + cat /tmp/tmp.lQKWpdmZ4M + rm /tmp/tmp.1DUmmEYBXK /tmp/tmp.lQKWpdmZ4M + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + xargs kubectl delete ns ++ mktemp ++ mktemp + awk '{print$1}' + local LAST_OUT=/tmp/tmp.MOIqKODHD3 ++ mktemp + local LAST_OUT=/tmp/tmp.d7vpomuYOU ++ mktemp + local LAST_ERR=/tmp/tmp.lwqjK5TRM4 + local exit_status=0 + local LAST_ERR=/tmp/tmp.fU8jGvnkgH + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.d7vpomuYOU + cat /tmp/tmp.fU8jGvnkgH + rm /tmp/tmp.d7vpomuYOU /tmp/tmp.fU8jGvnkgH + return 0 namespace "scheduled-backup-12053" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MOIqKODHD3 namespace "pxc-operator" deleted + cat /tmp/tmp.lwqjK5TRM4 + rm /tmp/tmp.MOIqKODHD3 /tmp/tmp.lwqjK5TRM4 + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.PB0F5X4RiF ++ mktemp + local LAST_ERR=/tmp/tmp.ZXI8oMftt8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PB0F5X4RiF namespace/pxc-operator created + cat /tmp/tmp.ZXI8oMftt8 + rm /tmp/tmp.PB0F5X4RiF /tmp/tmp.ZXI8oMftt8 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.X4xsX8YW4l +++ mktemp ++ local LAST_ERR=/tmp/tmp.YWqHWzrrtm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.X4xsX8YW4l ++ cat /tmp/tmp.YWqHWzrrtm ++ rm /tmp/tmp.X4xsX8YW4l /tmp/tmp.YWqHWzrrtm ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster4 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.zWYXQ5aD9f ++ mktemp + local LAST_ERR=/tmp/tmp.X7c5mfSms3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster4 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zWYXQ5aD9f Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster4" modified. + cat /tmp/tmp.X7c5mfSms3 + rm /tmp/tmp.zWYXQ5aD9f /tmp/tmp.X7c5mfSms3 + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.MiG5POthDm ++ mktemp + local LAST_ERR=/tmp/tmp.T5yAy7UPnr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MiG5POthDm customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.T5yAy7UPnr + rm /tmp/tmp.MiG5POthDm /tmp/tmp.T5yAy7UPnr + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/deploy/cw-rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.PBI4LnbSLS ++ mktemp + local LAST_ERR=/tmp/tmp.k5dMeM0CnK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PBI4LnbSLS clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.k5dMeM0CnK + rm /tmp/tmp.PBI4LnbSLS /tmp/tmp.k5dMeM0CnK + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - ++ mktemp + local LAST_OUT=/tmp/tmp.kVDIUw4xUR ++ mktemp + local LAST_ERR=/tmp/tmp.ue25qgu9WC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kVDIUw4xUR deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.ue25qgu9WC + rm /tmp/tmp.kVDIUw4xUR /tmp/tmp.ue25qgu9WC + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.6OyZJo30hu ++ mktemp + local LAST_ERR=/tmp/tmp.z9nGA22c7I + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6OyZJo30hu pod/percona-xtradb-cluster-operator-84658c6c47-d47wp condition met + cat /tmp/tmp.z9nGA22c7I + rm /tmp/tmp.6OyZJo30hu /tmp/tmp.z9nGA22c7I + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.D8Hmz9Qli8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.S9DgcQIXTp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.D8Hmz9Qli8 ++ cat /tmp/tmp.S9DgcQIXTp ++ rm /tmp/tmp.D8Hmz9Qli8 /tmp/tmp.S9DgcQIXTp ++ return 0 + wait_pod percona-xtradb-cluster-operator-84658c6c47-d47wp 480 pxc-operator + local pod=percona-xtradb-cluster-operator-84658c6c47-d47wp + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-84658c6c47-d47wp ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-84658c6c47-d47wp condition met waiting for pod/percona-xtradb-cluster-operator-84658c6c47-d47wp to become Ready.Ok + sleep 3 + create_namespace scheduled-backup-15195 + local namespace=scheduled-backup-15195 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ grep chaos-mesh.org ++ awk '{print $1}' ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces scheduled-backup-15195' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces scheduled-backup-15195 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace scheduled-backup-15195 + awk '{print$1}' ++ mktemp + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.gGbtCfVsLd + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_ERR=/tmp/tmp.5y0j1eHbjN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-15195 + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.SurBZw307t ++ mktemp + local LAST_ERR=/tmp/tmp.NPwEzyALYX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-15195 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SurBZw307t + cat /tmp/tmp.NPwEzyALYX + rm /tmp/tmp.SurBZw307t /tmp/tmp.NPwEzyALYX + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-15195 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.gGbtCfVsLd + cat /tmp/tmp.5y0j1eHbjN Error from server (NotFound): namespaces "scheduled-backup-15195" not found + rm /tmp/tmp.gGbtCfVsLd /tmp/tmp.5y0j1eHbjN + return 1 + : + wait_for_delete namespace/scheduled-backup-15195 + local res=namespace/scheduled-backup-15195 + echo -n 'waiting for namespace/scheduled-backup-15195 to be deleted' waiting for namespace/scheduled-backup-15195 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "scheduled-backup-15195" not found + desc 'create namespace scheduled-backup-15195' + set +o xtrace ----------------------------------------------------------------------------------- create namespace scheduled-backup-15195 ----------------------------------------------------------------------------------- + kubectl_bin create namespace scheduled-backup-15195 ++ mktemp + local LAST_OUT=/tmp/tmp.ncnpaIJfbx ++ mktemp + local LAST_ERR=/tmp/tmp.3tbqG7vgh7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace scheduled-backup-15195 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ncnpaIJfbx namespace/scheduled-backup-15195 created + cat /tmp/tmp.3tbqG7vgh7 + rm /tmp/tmp.ncnpaIJfbx /tmp/tmp.3tbqG7vgh7 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.toj87iaWo5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.hmQYTGczXp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.toj87iaWo5 ++ cat /tmp/tmp.hmQYTGczXp ++ rm /tmp/tmp.toj87iaWo5 /tmp/tmp.hmQYTGczXp ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster4 --namespace=scheduled-backup-15195 ++ mktemp + local LAST_OUT=/tmp/tmp.IfOsABYZvH ++ mktemp + local LAST_ERR=/tmp/tmp.s2tT28zy7K + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster4 --namespace=scheduled-backup-15195 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IfOsABYZvH Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster4" modified. + cat /tmp/tmp.s2tT28zy7K + rm /tmp/tmp.IfOsABYZvH /tmp/tmp.s2tT28zy7K + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.VWSNDhk9nZ ++ mktemp + local LAST_ERR=/tmp/tmp.oHwHPSf4l3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VWSNDhk9nZ secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.oHwHPSf4l3 + rm /tmp/tmp.VWSNDhk9nZ /tmp/tmp.oHwHPSf4l3 + return 0 + start_minio + deploy_helm scheduled-backup-15195 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "hashicorp" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Thu Dec 4 12:41:34 2025 NAMESPACE: scheduled-backup-15195 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.scheduled-backup-15195.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-15195 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-15195 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-15195 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-15195 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.A8hipDGRRM +++ mktemp ++ local LAST_ERR=/tmp/tmp.hvGl1cghvw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A8hipDGRRM ++ cat /tmp/tmp.hvGl1cghvw ++ rm /tmp/tmp.A8hipDGRRM /tmp/tmp.hvGl1cghvw ++ return 0 + MINIO_POD=minio-service-55fcc5d75f-smtcd + wait_pod minio-service-55fcc5d75f-smtcd + local pod=minio-service-55fcc5d75f-smtcd + local max_retry=480 + local ns= ++ echo minio-service-55fcc5d75f-smtcd ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minio-service-55fcc5d75f-smtcd condition met waiting for pod/minio-service-55fcc5d75f-smtcd to become Ready.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.gMYHAfkJsU ++ mktemp + local LAST_ERR=/tmp/tmp.NCMpQ4jOp3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gMYHAfkJsU make_bucket: operator-testing pod "aws-cli" deleted from scheduled-backup-15195 namespace + cat /tmp/tmp.NCMpQ4jOp3 All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_scheduled-backup-15195 + rm /tmp/tmp.gMYHAfkJsU /tmp/tmp.NCMpQ4jOp3 + return 0 + cluster=scheduled-backup + cat - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.B93qnrnLb2 ++ mktemp + local LAST_ERR=/tmp/tmp.J7WH91gzPp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.B93qnrnLb2 priorityclass.scheduling.k8s.io/high-priority configured + cat /tmp/tmp.J7WH91gzPp + rm /tmp/tmp.B93qnrnLb2 /tmp/tmp.J7WH91gzPp + return 0 + spinup_pxc scheduled-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local cluster=scheduled-backup + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.XcPMbOigXy ++ mktemp + local LAST_ERR=/tmp/tmp.4raM7oOxYq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XcPMbOigXy secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.4raM7oOxYq + rm /tmp/tmp.XcPMbOigXy /tmp/tmp.4raM7oOxYq + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/client.yml + local pvc_name= + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-15195~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.qSnVaS5r2t ++ mktemp + local LAST_ERR=/tmp/tmp.MysT0nmCM0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qSnVaS5r2t deployment.apps/pxc-client created + cat /tmp/tmp.MysT0nmCM0 + rm /tmp/tmp.qSnVaS5r2t /tmp/tmp.MysT0nmCM0 + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-15195~ + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + local LAST_OUT=/tmp/tmp.hRqmfjMWk3 ++ mktemp + local LAST_ERR=/tmp/tmp.r9W1A8cA4g + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hRqmfjMWk3 perconaxtradbcluster.pxc.percona.com/scheduled-backup created + cat /tmp/tmp.r9W1A8cA4g + rm /tmp/tmp.hRqmfjMWk3 /tmp/tmp.r9W1A8cA4g + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy scheduled-backup ++ local target_cluster=scheduled-backup +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.atmeFmNb0H ++++ mktemp +++ local LAST_ERR=/tmp/tmp.IKpD1tTnSA +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.atmeFmNb0H +++ cat /tmp/tmp.IKpD1tTnSA +++ rm /tmp/tmp.atmeFmNb0H /tmp/tmp.IKpD1tTnSA +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cj7TJCQmKG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yUKoOm3KYp +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.cj7TJCQmKG +++ cat /tmp/tmp.yUKoOm3KYp +++ rm /tmp/tmp.cj7TJCQmKG /tmp/tmp.yUKoOm3KYp +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scheduled-backup-proxysql ++ return + local proxy=scheduled-backup-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-15195 ++ mktemp + local LAST_OUT=/tmp/tmp.5cHd3ytqcM ++ mktemp + local LAST_ERR=/tmp/tmp.FtwfbgEuMM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-15195 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-15195 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-15195 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.5cHd3ytqcM + cat /tmp/tmp.FtwfbgEuMM error: no matching resources found + rm /tmp/tmp.5cHd3ytqcM /tmp/tmp.FtwfbgEuMM + return 1 + true + wait_for_running scheduled-backup-proxysql 1 + local name=scheduled-backup-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-proxysql-0 480 + local pod=scheduled-backup-proxysql-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-proxysql-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace pod/scheduled-backup-proxysql-0 condition met waiting for pod/scheduled-backup-proxysql-0 to become Ready.Ok + wait_for_running scheduled-backup-pxc 3 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 480 + local pod=scheduled-backup-pxc-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-0 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 480 + local pod=scheduled-backup-pxc-1 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 480 + local pod=scheduled-backup-pxc-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.NL8hIW9AiG +++ mktemp ++ local LAST_ERR=/tmp/tmp.tF5sErtxG9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NL8hIW9AiG ++ cat /tmp/tmp.tF5sErtxG9 ++ rm /tmp/tmp.NL8hIW9AiG /tmp/tmp.tF5sErtxG9 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ grep -E -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4WLPeopO0h +++ mktemp ++ local LAST_ERR=/tmp/tmp.vIgTr4RQNg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4WLPeopO0h ++ cat /tmp/tmp.vIgTr4RQNg Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.4WLPeopO0h /tmp/tmp.vIgTr4RQNg ++ return 0 ++ return 1 + [[ -n '' ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vz2xYr0yeZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.6rrgzKyO7N ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vz2xYr0yeZ ++ cat /tmp/tmp.6rrgzKyO7N ++ rm /tmp/tmp.vz2xYr0yeZ /tmp/tmp.6rrgzKyO7N ++ return 0 + client_pod=pxc-client-857d976497-6kb7x + wait_pod pxc-client-857d976497-6kb7x + local pod=pxc-client-857d976497-6kb7x + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-6kb7x ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-857d976497-6kb7x condition met waiting for pod/pxc-client-857d976497-6kb7x to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mqa9dHu0vE +++ mktemp ++ local LAST_ERR=/tmp/tmp.2U37ujzFWN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mqa9dHu0vE ++ cat /tmp/tmp.2U37ujzFWN ++ rm /tmp/tmp.mqa9dHu0vE /tmp/tmp.2U37ujzFWN ++ return 0 + client_pod=pxc-client-857d976497-6kb7x + wait_pod pxc-client-857d976497-6kb7x + local pod=pxc-client-857d976497-6kb7x + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-6kb7x ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-857d976497-6kb7x condition met waiting for pod/pxc-client-857d976497-6kb7x to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FKkg0662e8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.oj0vpdmNQj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FKkg0662e8 ++ cat /tmp/tmp.oj0vpdmNQj ++ rm /tmp/tmp.FKkg0662e8 /tmp/tmp.oj0vpdmNQj ++ return 0 + client_pod=pxc-client-857d976497-6kb7x + wait_pod pxc-client-857d976497-6kb7x + local pod=pxc-client-857d976497-6kb7x + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-6kb7x ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-857d976497-6kb7x condition met waiting for pod/pxc-client-857d976497-6kb7x to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.FSCaeHWrtq/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.FSCaeHWrtq/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.FSCaeHWrtq/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tnu5yBaaqy +++ mktemp ++ local LAST_ERR=/tmp/tmp.ibsXiRwC3f ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tnu5yBaaqy ++ cat /tmp/tmp.ibsXiRwC3f ++ rm /tmp/tmp.tnu5yBaaqy /tmp/tmp.ibsXiRwC3f ++ return 0 + client_pod=pxc-client-857d976497-6kb7x + wait_pod pxc-client-857d976497-6kb7x + local pod=pxc-client-857d976497-6kb7x + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-6kb7x ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-857d976497-6kb7x condition met waiting for pod/pxc-client-857d976497-6kb7x to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.FSCaeHWrtq/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.FSCaeHWrtq/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.FSCaeHWrtq/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eqxWcZaFa0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.sbS5u7PGXR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eqxWcZaFa0 ++ cat /tmp/tmp.sbS5u7PGXR ++ rm /tmp/tmp.eqxWcZaFa0 /tmp/tmp.sbS5u7PGXR ++ return 0 + client_pod=pxc-client-857d976497-6kb7x + wait_pod pxc-client-857d976497-6kb7x + local pod=pxc-client-857d976497-6kb7x + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-6kb7x ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-857d976497-6kb7x condition met waiting for pod/pxc-client-857d976497-6kb7x to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.FSCaeHWrtq/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.FSCaeHWrtq/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.FSCaeHWrtq/select-1.sql + is_keyring_plugin_in_use scheduled-backup + local cluster=scheduled-backup + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + grep -E -o 'early-plugin-load=keyring_\w+.so' ++ mktemp + local LAST_OUT=/tmp/tmp.XotmZG3Tyj ++ mktemp + local LAST_ERR=/tmp/tmp.HzvsmWqEXp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XotmZG3Tyj + cat /tmp/tmp.HzvsmWqEXp Unable to use a TTY - input is not a terminal or the right kind of file + rm /tmp/tmp.XotmZG3Tyj /tmp/tmp.HzvsmWqEXp + return 0 + return 1 + sleep 20 + desc 'add backups schedule for pvc storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for pvc storage ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.3DcCkZ3XqA +++ mktemp ++ local LAST_ERR=/tmp/tmp.pGnUSMLbfU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3DcCkZ3XqA ++ cat /tmp/tmp.pGnUSMLbfU ++ rm /tmp/tmp.3DcCkZ3XqA /tmp/tmp.pGnUSMLbfU ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster4 --namespace=scheduled-backup-15195 ++ mktemp + local LAST_OUT=/tmp/tmp.EjdJ8Y7HqZ ++ mktemp + local LAST_ERR=/tmp/tmp.ZtoPaGbGmM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster4 --namespace=scheduled-backup-15195 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EjdJ8Y7HqZ Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2076-ee1be0ba-8-cluster4" modified. + cat /tmp/tmp.ZtoPaGbGmM + rm /tmp/tmp.EjdJ8Y7HqZ /tmp/tmp.ZtoPaGbGmM + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-15195~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.sAUgAclJf9 + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + local LAST_ERR=/tmp/tmp.OAjmi51vvN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sAUgAclJf9 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.OAjmi51vvN + rm /tmp/tmp.sAUgAclJf9 /tmp/tmp.OAjmi51vvN + return 0 + label_node ++ kubectl_bin get nodes --no-headers=true ++ grep -v master ++ head -n1 +++ mktemp ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.BiP26yB0Ud +++ mktemp ++ local LAST_ERR=/tmp/tmp.NX86PLOOXe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get nodes --no-headers=true ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BiP26yB0Ud ++ cat /tmp/tmp.NX86PLOOXe ++ rm /tmp/tmp.BiP26yB0Ud /tmp/tmp.NX86PLOOXe ++ return 0 + LABELED_NODE=gke-jen-pxc-2076-ee1be0b-default-pool-65db86a9-747f + kubectl_bin label nodes gke-jen-pxc-2076-ee1be0b-default-pool-65db86a9-747f backupWorker=True --overwrite ++ mktemp + local LAST_OUT=/tmp/tmp.6E2tybNbYc ++ mktemp + local LAST_ERR=/tmp/tmp.T6rRdHfjne + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label nodes gke-jen-pxc-2076-ee1be0b-default-pool-65db86a9-747f backupWorker=True --overwrite + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6E2tybNbYc node/gke-jen-pxc-2076-ee1be0b-default-pool-65db86a9-747f not labeled + cat /tmp/tmp.T6rRdHfjne + rm /tmp/tmp.6E2tybNbYc /tmp/tmp.T6rRdHfjne + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-15195~ + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_OUT=/tmp/tmp.oR6GIW9z29 + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba#' + local LAST_ERR=/tmp/tmp.gcZe7Tje97 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oR6GIW9z29 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.gcZe7Tje97 + rm /tmp/tmp.oR6GIW9z29 /tmp/tmp.gcZe7Tje97 + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.NSoHKgqbx7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.6EdVKB8TDV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NSoHKgqbx7 ++ cat /tmp/tmp.6EdVKB8TDV ++ rm /tmp/tmp.NSoHKgqbx7 /tmp/tmp.6EdVKB8TDV ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.aOKVylgn2m +++ mktemp ++ local LAST_ERR=/tmp/tmp.dpsFlfle7h ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aOKVylgn2m ++ cat /tmp/tmp.dpsFlfle7h ++ rm /tmp/tmp.aOKVylgn2m /tmp/tmp.dpsFlfle7h ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.d3j9eXABZ5 ++ mktemp + local LAST_ERR=/tmp/tmp.Sy83MPFPW3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.d3j9eXABZ5 NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-2025124124912-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2025124124912-q6fav-20251204124912-0f2f44bb Running 34s + cat /tmp/tmp.Sy83MPFPW3 + rm /tmp/tmp.d3j9eXABZ5 /tmp/tmp.Sy83MPFPW3 + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.6OPe1QqIno ++ mktemp + local LAST_ERR=/tmp/tmp.s4Tyum2NAw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6OPe1QqIno NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-smtcd 1/1 Running 0 8m11s pxc-client-857d976497-6kb7x 2/2 Running 0 7m1s scheduled-backup-proxysql-0 3/3 Running 0 6m58s scheduled-backup-proxysql-1 3/3 Running 0 6m40s scheduled-backup-pxc-0 1/1 Running 0 6m58s scheduled-backup-pxc-1 1/1 Running 0 5m47s scheduled-backup-pxc-2 1/1 Running 0 4m37s xb-cron-scheduled-backup-pvc-2025124124912-q6fav-d42bm 0/1 Completed 0 36s + cat /tmp/tmp.s4Tyum2NAw + rm /tmp/tmp.6OPe1QqIno /tmp/tmp.s4Tyum2NAw + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Wtvk0zxNBY +++ mktemp ++ local LAST_ERR=/tmp/tmp.F4qyOvLw7p ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Wtvk0zxNBY ++ cat /tmp/tmp.F4qyOvLw7p ++ rm /tmp/tmp.Wtvk0zxNBY /tmp/tmp.F4qyOvLw7p ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.f0kvUTy5Yq +++ mktemp ++ local LAST_ERR=/tmp/tmp.MUTfh9UigX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.f0kvUTy5Yq ++ cat /tmp/tmp.MUTfh9UigX ++ rm /tmp/tmp.f0kvUTy5Yq /tmp/tmp.MUTfh9UigX ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-15195~ + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_OUT=/tmp/tmp.g5aVh8s5jX ++ mktemp + local LAST_ERR=/tmp/tmp.EHkR0jLfS4 + local exit_status=0 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.g5aVh8s5jX perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.EHkR0jLfS4 + rm /tmp/tmp.g5aVh8s5jX /tmp/tmp.EHkR0jLfS4 + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + local pvc_name= + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2076/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2076-ee1be0ba#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-15195~ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + local LAST_OUT=/tmp/tmp.xLRuQoPKeG + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + local LAST_ERR=/tmp/tmp.YejU5v4vem + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xLRuQoPKeG perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.YejU5v4vem + rm /tmp/tmp.xLRuQoPKeG /tmp/tmp.YejU5v4vem + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.LaIAZd0eiX +++ mktemp ++ local LAST_ERR=/tmp/tmp.9y47WbyM6E ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LaIAZd0eiX ++ cat /tmp/tmp.9y47WbyM6E ++ rm /tmp/tmp.LaIAZd0eiX /tmp/tmp.9y47WbyM6E ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.ouIquZbgE9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.TBmcN2X9LZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ouIquZbgE9 ++ cat /tmp/tmp.TBmcN2X9LZ ++ rm /tmp/tmp.ouIquZbgE9 /tmp/tmp.TBmcN2X9LZ ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-1 + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.tJhguRUMn5 ++ mktemp + local LAST_ERR=/tmp/tmp.6JFaT6FAge + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tJhguRUMn5 NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125113-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:51:13-full Running 31s cron-scheduled-backup-pvc-2025124124912-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2025124124912-q6fav-20251204124912-0f2f44bb Succeeded 115s 2m32s + cat /tmp/tmp.6JFaT6FAge + rm /tmp/tmp.tJhguRUMn5 /tmp/tmp.6JFaT6FAge + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.Jb7Z3l818o ++ mktemp + local LAST_ERR=/tmp/tmp.wQHFkIBa3t + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Jb7Z3l818o NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-smtcd 1/1 Running 0 10m pxc-client-857d976497-6kb7x 2/2 Running 0 8m57s scheduled-backup-proxysql-0 3/3 Running 0 8m54s scheduled-backup-proxysql-1 3/3 Running 0 8m36s scheduled-backup-pxc-0 1/1 Running 0 8m54s scheduled-backup-pxc-1 1/1 Running 0 7m43s scheduled-backup-pxc-2 1/1 Running 0 6m33s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-mdnbv 1/1 Running 0 31s xb-cron-scheduled-backup-pvc-2025124124912-q6fav-d42bm 0/1 Completed 0 2m32s + cat /tmp/tmp.wQHFkIBa3t + rm /tmp/tmp.Jb7Z3l818o /tmp/tmp.wQHFkIBa3t + return 0 + sleep 20 ++ get_running_backups_amount ++ wc -l ++ grep -vE ':Succeeded|:Failed' ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bI1FXo62pC +++ mktemp ++ local LAST_ERR=/tmp/tmp.CGHoQoweGC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bI1FXo62pC ++ cat /tmp/tmp.CGHoQoweGC ++ rm /tmp/tmp.bI1FXo62pC /tmp/tmp.CGHoQoweGC ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.iqqS5pD4gq +++ mktemp ++ local LAST_ERR=/tmp/tmp.nVq2wn1ADc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iqqS5pD4gq ++ cat /tmp/tmp.nVq2wn1ADc ++ rm /tmp/tmp.iqqS5pD4gq /tmp/tmp.nVq2wn1ADc ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-1 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.cdMm1wsBGC ++ mktemp + local LAST_ERR=/tmp/tmp.XqQeXe5qCV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cdMm1wsBGC NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125113-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:51:13-full Running 70s cron-scheduled-backup-pvc-2025124124912-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2025124124912-q6fav-20251204124912-0f2f44bb Succeeded 2m34s 3m11s + cat /tmp/tmp.XqQeXe5qCV + rm /tmp/tmp.cdMm1wsBGC /tmp/tmp.XqQeXe5qCV + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.JIxB8Jv3Z7 ++ mktemp + local LAST_ERR=/tmp/tmp.TUsnb8sU2Y + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JIxB8Jv3Z7 NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-smtcd 1/1 Running 0 10m pxc-client-857d976497-6kb7x 2/2 Running 0 9m37s scheduled-backup-proxysql-0 3/3 Running 0 9m34s scheduled-backup-proxysql-1 3/3 Running 0 9m16s scheduled-backup-pxc-0 1/1 Running 0 9m34s scheduled-backup-pxc-1 1/1 Running 0 8m23s scheduled-backup-pxc-2 1/1 Running 0 7m13s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-mdnbv 0/1 Error 0 71s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-slt96 1/1 Running 0 26s xb-cron-scheduled-backup-pvc-2025124124912-q6fav-d42bm 0/1 Completed 0 3m12s + cat /tmp/tmp.TUsnb8sU2Y + rm /tmp/tmp.JIxB8Jv3Z7 /tmp/tmp.TUsnb8sU2Y + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.ra2gKMCbLR +++ mktemp ++ local LAST_ERR=/tmp/tmp.dghM3TdIoG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ra2gKMCbLR ++ cat /tmp/tmp.dghM3TdIoG ++ rm /tmp/tmp.ra2gKMCbLR /tmp/tmp.dghM3TdIoG ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.FzPvDqrYQa +++ mktemp ++ local LAST_ERR=/tmp/tmp.gvb0rDxRc7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FzPvDqrYQa ++ cat /tmp/tmp.gvb0rDxRc7 ++ rm /tmp/tmp.FzPvDqrYQa /tmp/tmp.gvb0rDxRc7 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.Q4xtKeGS6p ++ mktemp + local LAST_ERR=/tmp/tmp.VQYGlY63WQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Q4xtKeGS6p NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125113-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:51:13-full Running 109s cron-scheduled-backup-pvc-2025124124912-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2025124124912-q6fav-20251204124912-0f2f44bb Succeeded 3m13s 3m50s + cat /tmp/tmp.VQYGlY63WQ + rm /tmp/tmp.Q4xtKeGS6p /tmp/tmp.VQYGlY63WQ + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.HoHgWQH95j ++ mktemp + local LAST_ERR=/tmp/tmp.qX1AowcM8M + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HoHgWQH95j NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-smtcd 1/1 Running 0 11m pxc-client-857d976497-6kb7x 2/2 Running 0 10m scheduled-backup-proxysql-0 3/3 Running 0 10m scheduled-backup-proxysql-1 3/3 Running 0 9m55s scheduled-backup-pxc-0 1/1 Running 0 10m scheduled-backup-pxc-1 1/1 Running 0 9m2s scheduled-backup-pxc-2 1/1 Running 0 7m52s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-mdnbv 0/1 Error 0 110s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-slt96 1/1 Running 0 65s xb-cron-scheduled-backup-pvc-2025124124912-q6fav-d42bm 0/1 Completed 0 3m51s + cat /tmp/tmp.qX1AowcM8M + rm /tmp/tmp.HoHgWQH95j /tmp/tmp.qX1AowcM8M + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.A8OsldIYK2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.x0ESDgXRZH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A8OsldIYK2 ++ cat /tmp/tmp.x0ESDgXRZH ++ rm /tmp/tmp.A8OsldIYK2 /tmp/tmp.x0ESDgXRZH ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.7vzs7mey8S +++ mktemp ++ local LAST_ERR=/tmp/tmp.cpP5pcl0HM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7vzs7mey8S ++ cat /tmp/tmp.cpP5pcl0HM ++ rm /tmp/tmp.7vzs7mey8S /tmp/tmp.cpP5pcl0HM ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-2 + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.h54oMCa3j3 ++ mktemp + local LAST_ERR=/tmp/tmp.s7c8kbEkWP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.h54oMCa3j3 NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125113-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:51:13-full Starting 2m32s cron-scheduled-backup-pvc-2025124124912-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2025124124912-q6fav-20251204124912-0f2f44bb Succeeded 3m56s 4m33s + cat /tmp/tmp.s7c8kbEkWP + rm /tmp/tmp.h54oMCa3j3 /tmp/tmp.s7c8kbEkWP + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.wQ411D3LQ2 ++ mktemp + local LAST_ERR=/tmp/tmp.oJSGePc1XC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wQ411D3LQ2 NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-smtcd 1/1 Running 0 12m pxc-client-857d976497-6kb7x 2/2 Running 0 10m scheduled-backup-proxysql-0 3/3 Running 0 10m scheduled-backup-proxysql-1 3/3 Running 0 10m scheduled-backup-pxc-0 1/1 Running 0 10m scheduled-backup-pxc-1 1/1 Running 0 9m45s scheduled-backup-pxc-2 1/1 Running 0 8m35s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-mdnbv 0/1 Error 0 2m33s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-slt96 0/1 Error 0 108s xb-cron-scheduled-backup-pvc-2025124124912-q6fav-d42bm 0/1 Completed 0 4m34s + cat /tmp/tmp.oJSGePc1XC + rm /tmp/tmp.wQ411D3LQ2 /tmp/tmp.oJSGePc1XC + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.uBmo8bzWjT +++ mktemp ++ local LAST_ERR=/tmp/tmp.Yv1wb7mXn9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uBmo8bzWjT ++ cat /tmp/tmp.Yv1wb7mXn9 ++ rm /tmp/tmp.uBmo8bzWjT /tmp/tmp.Yv1wb7mXn9 ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.9lowI2Q24l +++ mktemp ++ local LAST_ERR=/tmp/tmp.4SG6923W5i ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9lowI2Q24l ++ cat /tmp/tmp.4SG6923W5i ++ rm /tmp/tmp.9lowI2Q24l /tmp/tmp.4SG6923W5i ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-2 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.FMRlQGLzzx ++ mktemp + local LAST_ERR=/tmp/tmp.Mcu3zvveZE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FMRlQGLzzx NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125113-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:51:13-full Running 3m20s cron-scheduled-backup-pvc-2025124124912-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2025124124912-q6fav-20251204124912-0f2f44bb Succeeded 4m44s 5m21s + cat /tmp/tmp.Mcu3zvveZE + rm /tmp/tmp.FMRlQGLzzx /tmp/tmp.Mcu3zvveZE + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.tUtLqxNr7u ++ mktemp + local LAST_ERR=/tmp/tmp.0XreSWruPd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tUtLqxNr7u NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-smtcd 1/1 Running 0 12m pxc-client-857d976497-6kb7x 2/2 Running 0 11m scheduled-backup-proxysql-0 3/3 Running 0 11m scheduled-backup-proxysql-1 3/3 Running 0 11m scheduled-backup-pxc-0 1/1 Running 0 11m scheduled-backup-pxc-1 1/1 Running 0 10m scheduled-backup-pxc-2 1/1 Running 0 9m22s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-gn8tp 1/1 Running 0 40s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-mdnbv 0/1 Error 0 3m20s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-slt96 0/1 Error 0 2m35s xb-cron-scheduled-backup-pvc-2025124124912-q6fav-d42bm 0/1 Completed 0 5m21s + cat /tmp/tmp.0XreSWruPd + rm /tmp/tmp.tUtLqxNr7u /tmp/tmp.0XreSWruPd + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.g2cxFhAFhG +++ mktemp ++ local LAST_ERR=/tmp/tmp.At9kLkfUZ0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.g2cxFhAFhG ++ cat /tmp/tmp.At9kLkfUZ0 ++ rm /tmp/tmp.g2cxFhAFhG /tmp/tmp.At9kLkfUZ0 ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ grep :Failed ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KCWMCKjZB8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z3onCwW8pW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KCWMCKjZB8 ++ cat /tmp/tmp.Z3onCwW8pW ++ rm /tmp/tmp.KCWMCKjZB8 /tmp/tmp.Z3onCwW8pW ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.M1WHgSGC9e ++ mktemp + local LAST_ERR=/tmp/tmp.aEUrhxpohg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.M1WHgSGC9e NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125113-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:51:13-full Running 4m3s cron-scheduled-backup-pvc-2025124124912-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2025124124912-q6fav-20251204124912-0f2f44bb Succeeded 5m27s 6m4s + cat /tmp/tmp.aEUrhxpohg + rm /tmp/tmp.M1WHgSGC9e /tmp/tmp.aEUrhxpohg + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.VzCp9Lrdpj ++ mktemp + local LAST_ERR=/tmp/tmp.Kp5bP6coim + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VzCp9Lrdpj NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-smtcd 1/1 Running 0 13m pxc-client-857d976497-6kb7x 2/2 Running 0 12m scheduled-backup-proxysql-0 3/3 Running 0 12m scheduled-backup-proxysql-1 3/3 Running 0 12m scheduled-backup-pxc-0 1/1 Running 0 12m scheduled-backup-pxc-1 1/1 Running 0 11m scheduled-backup-pxc-2 1/1 Running 0 10m xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-gn8tp 1/1 Running 0 83s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-mdnbv 0/1 Error 0 4m3s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-slt96 0/1 Error 0 3m18s xb-cron-scheduled-backup-pvc-2025124124912-q6fav-d42bm 0/1 Completed 0 6m4s + cat /tmp/tmp.Kp5bP6coim + rm /tmp/tmp.VzCp9Lrdpj /tmp/tmp.Kp5bP6coim + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ wc -l ++ grep -vE ':Succeeded|:Failed' ++ local LAST_OUT=/tmp/tmp.SvLqFl4c7c +++ mktemp ++ local LAST_ERR=/tmp/tmp.fLLkRmZo76 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SvLqFl4c7c ++ cat /tmp/tmp.fLLkRmZo76 ++ rm /tmp/tmp.SvLqFl4c7c /tmp/tmp.fLLkRmZo76 ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.11WBERXqiB +++ mktemp ++ local LAST_ERR=/tmp/tmp.ia5Q8a9m0C ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.11WBERXqiB ++ cat /tmp/tmp.ia5Q8a9m0C ++ rm /tmp/tmp.11WBERXqiB /tmp/tmp.ia5Q8a9m0C ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.PKKh5Hquk1 ++ mktemp + local LAST_ERR=/tmp/tmp.rcLCVgbdd8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PKKh5Hquk1 NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125113-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:51:13-full Starting 4m47s cron-scheduled-backup-pvc-2025124124912-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2025124124912-q6fav-20251204124912-0f2f44bb Succeeded 6m11s 6m48s + cat /tmp/tmp.rcLCVgbdd8 + rm /tmp/tmp.PKKh5Hquk1 /tmp/tmp.rcLCVgbdd8 + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.55WSsFq3DC ++ mktemp + local LAST_ERR=/tmp/tmp.m7TsWN9slm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.55WSsFq3DC NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-smtcd 1/1 Running 0 14m pxc-client-857d976497-6kb7x 2/2 Running 0 13m scheduled-backup-proxysql-0 3/3 Running 0 13m scheduled-backup-proxysql-1 3/3 Running 0 12m scheduled-backup-pxc-0 1/1 Running 0 13m scheduled-backup-pxc-1 1/1 Running 0 12m scheduled-backup-pxc-2 1/1 Running 0 10m xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-gn8tp 0/1 Error 0 2m8s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-mdnbv 0/1 Error 0 4m48s xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-slt96 0/1 Error 0 4m3s xb-cron-scheduled-backup-pvc-2025124124912-q6fav-d42bm 0/1 Completed 0 6m49s + cat /tmp/tmp.m7TsWN9slm + rm /tmp/tmp.55WSsFq3DC /tmp/tmp.m7TsWN9slm + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fBzJO9wX0j +++ mktemp ++ local LAST_ERR=/tmp/tmp.aGXcFn8LcT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fBzJO9wX0j ++ cat /tmp/tmp.aGXcFn8LcT ++ rm /tmp/tmp.fBzJO9wX0j /tmp/tmp.aGXcFn8LcT ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.BArBWcubZm +++ mktemp ++ local LAST_ERR=/tmp/tmp.kawsdr60sy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BArBWcubZm ++ cat /tmp/tmp.kawsdr60sy ++ rm /tmp/tmp.BArBWcubZm /tmp/tmp.kawsdr60sy ++ return 0 + [[ 1 -gt 0 ]] + echo 'One or more backups have been failed!\n' One or more backups have been failed!\n + desc 'LIST OF BACKUPS' + set +o xtrace ----------------------------------------------------------------------------------- LIST OF BACKUPS ----------------------------------------------------------------------------------- + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.WUYtdu4rM7 ++ mktemp + local LAST_ERR=/tmp/tmp.qtEQS1Hl9J + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WUYtdu4rM7 NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2025124125113-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-12-04-12:51:13-full Failed 5m12s cron-scheduled-backup-pvc-2025124124912-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2025124124912-q6fav-20251204124912-0f2f44bb Succeeded 6m36s 7m13s + cat /tmp/tmp.qtEQS1Hl9J + rm /tmp/tmp.WUYtdu4rM7 /tmp/tmp.qtEQS1Hl9J + return 0 + desc 'LIST OF PODS' + set +o xtrace ----------------------------------------------------------------------------------- LIST OF PODS ----------------------------------------------------------------------------------- + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.5FoqmQL1rU ++ mktemp + local LAST_ERR=/tmp/tmp.fIg1EmhR7C + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5FoqmQL1rU NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-smtcd 1/1 Running 0 14m pxc-client-857d976497-6kb7x 2/2 Running 0 13m scheduled-backup-proxysql-0 3/3 Running 0 13m scheduled-backup-proxysql-1 3/3 Running 0 13m scheduled-backup-pxc-0 1/1 Running 0 13m scheduled-backup-pxc-1 1/1 Running 0 12m scheduled-backup-pxc-2 1/1 Running 0 11m xb-cron-scheduled-backup-aws-s3-2025124125113-q6fav-dqg5d 1/1 Terminating 0 17s xb-cron-scheduled-backup-pvc-2025124124912-q6fav-d42bm 0/1 Completed 0 7m13s + cat /tmp/tmp.fIg1EmhR7C + rm /tmp/tmp.5FoqmQL1rU /tmp/tmp.fIg1EmhR7C + return 0 + exit 1