Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/logs/scheduled-backup-5-7.log WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 + main + create_infra scheduled-backup-19247 + local ns=scheduled-backup-19247 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n scheduled-backup-23843 scheduled-backup --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.GInLavdVql ++ mktemp + local LAST_ERR=/tmp/tmp.L4ftBfTFsG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GInLavdVql perconaxtradbcluster.pxc.percona.com "scheduled-backup" deleted + cat /tmp/tmp.L4ftBfTFsG + rm /tmp/tmp.GInLavdVql /tmp/tmp.L4ftBfTFsG + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.KxcAkvg17L ++ mktemp + local LAST_ERR=/tmp/tmp.ebfjoF2pE2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KxcAkvg17L perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-aws-s3-2024514164851-q6fav" deleted perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-azure-blob-2024514164653-q6fav" deleted perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-gcp-cs-2024514164456-q6fav" deleted perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-minio-2024514163917-q6fav" deleted perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-pvc-2024514163445-q6fav" deleted + cat /tmp/tmp.ebfjoF2pE2 + rm /tmp/tmp.KxcAkvg17L /tmp/tmp.ebfjoF2pE2 + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.v8a7fmbmRi ++ mktemp + local LAST_ERR=/tmp/tmp.MXIJ1hyCu1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.v8a7fmbmRi perconaxtradbclusterrestore.pxc.percona.com "aws-s3-2024514164851-q6fav" deleted perconaxtradbclusterrestore.pxc.percona.com "gcp-cs-2024514164456-q6fav" deleted perconaxtradbclusterrestore.pxc.percona.com "minio-2024514163917-q6fav" deleted perconaxtradbclusterrestore.pxc.percona.com "pvc-2024514163445-q6fav" deleted + cat /tmp/tmp.MXIJ1hyCu1 + rm /tmp/tmp.v8a7fmbmRi /tmp/tmp.MXIJ1hyCu1 + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.PEykSUwN0x + local LAST_OUT=/tmp/tmp.UmngcnsF6G ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.NeQfUyQJ7n + local exit_status=0 + local LAST_ERR=/tmp/tmp.kv41DU6abD + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UmngcnsF6G + cat /tmp/tmp.NeQfUyQJ7n + rm /tmp/tmp.UmngcnsF6G /tmp/tmp.NeQfUyQJ7n + return 0 namespace "scheduled-backup-23843" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PEykSUwN0x namespace "pxc-operator" deleted + cat /tmp/tmp.kv41DU6abD + rm /tmp/tmp.PEykSUwN0x /tmp/tmp.kv41DU6abD + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'namespace/pxc-operator - ' namespace/pxc-operator - + set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ihYwfSpeCZ ++ mktemp + local LAST_ERR=/tmp/tmp.K5F7tzaOZn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ihYwfSpeCZ namespace/pxc-operator created + cat /tmp/tmp.K5F7tzaOZn + rm /tmp/tmp.ihYwfSpeCZ /tmp/tmp.K5F7tzaOZn + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.FvZ0keYkaA +++ mktemp ++ local LAST_ERR=/tmp/tmp.iio0YSwExm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FvZ0keYkaA ++ cat /tmp/tmp.iio0YSwExm ++ rm /tmp/tmp.FvZ0keYkaA /tmp/tmp.iio0YSwExm ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1710-fb797906-3-cluster6 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.r343yBg8YA ++ mktemp + local LAST_ERR=/tmp/tmp.a2lXVBpsWT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1710-fb797906-3-cluster6 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.r343yBg8YA Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1710-fb797906-3-cluster6" modified. + cat /tmp/tmp.a2lXVBpsWT + rm /tmp/tmp.r343yBg8YA /tmp/tmp.a2lXVBpsWT + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.rf9bYbCv6W ++ mktemp + local LAST_ERR=/tmp/tmp.10K0VRzvXX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rf9bYbCv6W customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.10K0VRzvXX + rm /tmp/tmp.rf9bYbCv6W /tmp/tmp.10K0VRzvXX + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.jD9SiAJFJX ++ mktemp + local LAST_ERR=/tmp/tmp.3LatmQe6jB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jD9SiAJFJX clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.3LatmQe6jB + rm /tmp/tmp.jD9SiAJFJX /tmp/tmp.3LatmQe6jB + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - ++ mktemp + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + local LAST_OUT=/tmp/tmp.Vdpi1wnqYg ++ mktemp + local LAST_ERR=/tmp/tmp.WnLUHye1s6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Vdpi1wnqYg deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.WnLUHye1s6 + rm /tmp/tmp.Vdpi1wnqYg /tmp/tmp.WnLUHye1s6 + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.crAbhUf5S9 ++ mktemp + local LAST_ERR=/tmp/tmp.DJX77nmGcN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.crAbhUf5S9 pod/percona-xtradb-cluster-operator-8448ddf579-7lw5z condition met + cat /tmp/tmp.DJX77nmGcN + rm /tmp/tmp.crAbhUf5S9 /tmp/tmp.DJX77nmGcN + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.vRUOXCNJVl +++ mktemp ++ local LAST_ERR=/tmp/tmp.GA2xYK2fOk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vRUOXCNJVl ++ cat /tmp/tmp.GA2xYK2fOk ++ rm /tmp/tmp.vRUOXCNJVl /tmp/tmp.GA2xYK2fOk ++ return 0 + wait_pod percona-xtradb-cluster-operator-8448ddf579-7lw5z 480 pxc-operator + local pod=percona-xtradb-cluster-operator-8448ddf579-7lw5z + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-8448ddf579-7lw5z ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-8448ddf579-7lw5z condition met percona-xtradb-cluster-operator-8448ddf579-7lw5z.Ok + sleep 3 + create_namespace scheduled-backup-19247 + local namespace=scheduled-backup-19247 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces scheduled-backup-19247' + set +o xtrace ++ mktemp ----------------------------------------------------------------------------------- cleaned up old namespaces scheduled-backup-19247 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace scheduled-backup-19247 + awk '{print$1}' ++ mktemp + local LAST_OUT=/tmp/tmp.tOeG2c2g23 + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.oDXGmVMoCS ++ mktemp + local LAST_ERR=/tmp/tmp.vF5AMcpP5G + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.4yZ3fkZhmn + local exit_status=0 + for i in '$(seq 0 2)' + set +e ++ seq 0 2 + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-19247 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-19247 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tOeG2c2g23 + cat /tmp/tmp.vF5AMcpP5G + rm /tmp/tmp.tOeG2c2g23 /tmp/tmp.vF5AMcpP5G + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-19247 Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.oDXGmVMoCS + cat /tmp/tmp.4yZ3fkZhmn Error from server (NotFound): namespaces "scheduled-backup-19247" not found + rm /tmp/tmp.oDXGmVMoCS /tmp/tmp.4yZ3fkZhmn + return 1 + : + wait_for_delete namespace/scheduled-backup-19247 + local res=namespace/scheduled-backup-19247 + echo -n 'namespace/scheduled-backup-19247 - ' namespace/scheduled-backup-19247 - + set +o xtrace Error from server (NotFound): namespaces "scheduled-backup-19247" not found + desc 'create namespace scheduled-backup-19247' + set +o xtrace ----------------------------------------------------------------------------------- create namespace scheduled-backup-19247 ----------------------------------------------------------------------------------- + kubectl_bin create namespace scheduled-backup-19247 ++ mktemp + local LAST_OUT=/tmp/tmp.pvBa0bzHjq ++ mktemp + local LAST_ERR=/tmp/tmp.qvJ3Il4u46 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace scheduled-backup-19247 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pvBa0bzHjq namespace/scheduled-backup-19247 created + cat /tmp/tmp.qvJ3Il4u46 + rm /tmp/tmp.pvBa0bzHjq /tmp/tmp.qvJ3Il4u46 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.BnBrQUMqoe +++ mktemp ++ local LAST_ERR=/tmp/tmp.j0SSJqcoN7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BnBrQUMqoe ++ cat /tmp/tmp.j0SSJqcoN7 ++ rm /tmp/tmp.BnBrQUMqoe /tmp/tmp.j0SSJqcoN7 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1710-fb797906-3-cluster6 --namespace=scheduled-backup-19247 ++ mktemp + local LAST_OUT=/tmp/tmp.eHwxVszjCP ++ mktemp + local LAST_ERR=/tmp/tmp.Pp5XtYjwJa + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1710-fb797906-3-cluster6 --namespace=scheduled-backup-19247 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eHwxVszjCP Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1710-fb797906-3-cluster6" modified. + cat /tmp/tmp.Pp5XtYjwJa + rm /tmp/tmp.eHwxVszjCP /tmp/tmp.Pp5XtYjwJa + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.XzgMzLj3kj ++ mktemp + local LAST_ERR=/tmp/tmp.kwWkRBmK8M + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XzgMzLj3kj secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.kwWkRBmK8M + rm /tmp/tmp.XzgMzLj3kj /tmp/tmp.kwWkRBmK8M + return 0 + start_minio + deploy_helm scheduled-backup-19247 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "stable" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Tue May 14 17:27:40 2024 NAMESPACE: scheduled-backup-19247 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.scheduled-backup-19247.svc.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-19247 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-19247 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-19247 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-19247 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DXcxaMDAfm +++ mktemp ++ local LAST_ERR=/tmp/tmp.L4ZlcAfYSH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DXcxaMDAfm ++ cat /tmp/tmp.L4ZlcAfYSH ++ rm /tmp/tmp.DXcxaMDAfm /tmp/tmp.L4ZlcAfYSH ++ return 0 + MINIO_POD=minio-service-76ffcfd45-wgdxc + wait_pod minio-service-76ffcfd45-wgdxc + local pod=minio-service-76ffcfd45-wgdxc + local max_retry=480 + local ns= ++ echo minio-service-76ffcfd45-wgdxc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minio-service-76ffcfd45-wgdxc condition met minio-service-76ffcfd45-wgdxc.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.NQ5jN0Fm7h ++ mktemp + local LAST_ERR=/tmp/tmp.1LawPBKXjf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NQ5jN0Fm7h make_bucket: operator-testing pod "aws-cli" deleted + cat /tmp/tmp.1LawPBKXjf If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: unable to upgrade connection: container aws-cli not found in pod aws-cli_scheduled-backup-19247 + rm /tmp/tmp.NQ5jN0Fm7h /tmp/tmp.1LawPBKXjf + return 0 + cluster=scheduled-backup + cat - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.W3Yi25Tuwm ++ mktemp + local LAST_ERR=/tmp/tmp.sHmsUYPR7T + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.W3Yi25Tuwm priorityclass.scheduling.k8s.io/high-priority configured + cat /tmp/tmp.sHmsUYPR7T + rm /tmp/tmp.W3Yi25Tuwm /tmp/tmp.sHmsUYPR7T + return 0 + spinup_pxc scheduled-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local cluster=scheduled-backup + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.dnZQZjpkGv ++ mktemp + local LAST_ERR=/tmp/tmp.B45SzPCrn5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dnZQZjpkGv secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.B45SzPCrn5 + rm /tmp/tmp.dnZQZjpkGv /tmp/tmp.B45SzPCrn5 + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/conf/client.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/conf/client.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.Pbg4OPyA7W + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + local LAST_ERR=/tmp/tmp.1eQHnFJP4S + local exit_status=0 + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19247~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Pbg4OPyA7W deployment.apps/pxc-client created + cat /tmp/tmp.1eQHnFJP4S + rm /tmp/tmp.Pbg4OPyA7W /tmp/tmp.1eQHnFJP4S + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_OUT=/tmp/tmp.MWLxHiytSG + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19247~ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906#' + local LAST_ERR=/tmp/tmp.DQynmPCIbC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MWLxHiytSG perconaxtradbcluster.pxc.percona.com/scheduled-backup created + cat /tmp/tmp.DQynmPCIbC + rm /tmp/tmp.MWLxHiytSG /tmp/tmp.DQynmPCIbC + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy scheduled-backup ++ local target_cluster=scheduled-backup +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ne2gje25IB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.pzzXuwvPSK +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.ne2gje25IB +++ cat /tmp/tmp.pzzXuwvPSK +++ rm /tmp/tmp.ne2gje25IB /tmp/tmp.pzzXuwvPSK +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.x5vEZf1jIh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ofCU436F0g +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.x5vEZf1jIh +++ cat /tmp/tmp.ofCU436F0g +++ rm /tmp/tmp.x5vEZf1jIh /tmp/tmp.ofCU436F0g +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scheduled-backup-proxysql ++ return + local proxy=scheduled-backup-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-19247 ++ mktemp + local LAST_OUT=/tmp/tmp.0Awg4jiOhS ++ mktemp + local LAST_ERR=/tmp/tmp.6uLNr15E4v + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-19247 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-19247 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-19247 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.0Awg4jiOhS + cat /tmp/tmp.6uLNr15E4v error: no matching resources found + rm /tmp/tmp.0Awg4jiOhS /tmp/tmp.6uLNr15E4v + return 1 + true + wait_for_running scheduled-backup-proxysql 1 + local name=scheduled-backup-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-proxysql-0 480 + local pod=scheduled-backup-proxysql-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/scheduled-backup-proxysql-0 condition met scheduled-backup-proxysql-0.Ok + wait_for_running scheduled-backup-pxc 3 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 480 + local pod=scheduled-backup-pxc-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 480 + local pod=scheduled-backup-pxc-1 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 480 + local pod=scheduled-backup-pxc-2 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pBAsIJpSy7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.mVW9R6Jnsj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pBAsIJpSy7 ++ cat /tmp/tmp.mVW9R6Jnsj Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.pBAsIJpSy7 /tmp/tmp.mVW9R6Jnsj ++ return 0 + [[ -n '' ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scheduled-backup-proxysql -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h scheduled-backup-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.l25eypQGa5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.B0JgeNND3D ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.l25eypQGa5 ++ cat /tmp/tmp.B0JgeNND3D ++ rm /tmp/tmp.l25eypQGa5 /tmp/tmp.B0JgeNND3D ++ return 0 + client_pod=pxc-client-64b479df95-9r42p + wait_pod pxc-client-64b479df95-9r42p + local pod=pxc-client-64b479df95-9r42p + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-9r42p ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-9r42p condition met pxc-client-64b479df95-9r42p.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scheduled-backup-proxysql -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h scheduled-backup-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BGo0ZBMw6Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.PSDWjXoEls ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BGo0ZBMw6Q ++ cat /tmp/tmp.PSDWjXoEls ++ rm /tmp/tmp.BGo0ZBMw6Q /tmp/tmp.PSDWjXoEls ++ return 0 + client_pod=pxc-client-64b479df95-9r42p + wait_pod pxc-client-64b479df95-9r42p + local pod=pxc-client-64b479df95-9r42p + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-9r42p ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-9r42p condition met pxc-client-64b479df95-9r42p.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BacIS90wBB +++ mktemp ++ local LAST_ERR=/tmp/tmp.QQJN9IjQRl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BacIS90wBB ++ cat /tmp/tmp.QQJN9IjQRl ++ rm /tmp/tmp.BacIS90wBB /tmp/tmp.QQJN9IjQRl ++ return 0 + client_pod=pxc-client-64b479df95-9r42p + wait_pod pxc-client-64b479df95-9r42p + local pod=pxc-client-64b479df95-9r42p + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-9r42p ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-64b479df95-9r42p condition met pxc-client-64b479df95-9r42p.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.pD5gJ3NuC8/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.pD5gJ3NuC8/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aWH9GBVPJd +++ mktemp ++ local LAST_ERR=/tmp/tmp.rge8Pj0TtV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aWH9GBVPJd ++ cat /tmp/tmp.rge8Pj0TtV ++ rm /tmp/tmp.aWH9GBVPJd /tmp/tmp.rge8Pj0TtV ++ return 0 + client_pod=pxc-client-64b479df95-9r42p + wait_pod pxc-client-64b479df95-9r42p + local pod=pxc-client-64b479df95-9r42p + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-9r42p ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-9r42p condition met pxc-client-64b479df95-9r42p.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.pD5gJ3NuC8/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.pD5gJ3NuC8/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.B6FR2nKVAE +++ mktemp ++ local LAST_ERR=/tmp/tmp.VpLGYfRHPB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.B6FR2nKVAE ++ cat /tmp/tmp.VpLGYfRHPB ++ rm /tmp/tmp.B6FR2nKVAE /tmp/tmp.VpLGYfRHPB ++ return 0 + client_pod=pxc-client-64b479df95-9r42p + wait_pod pxc-client-64b479df95-9r42p + local pod=pxc-client-64b479df95-9r42p + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-9r42p ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-64b479df95-9r42p condition met pxc-client-64b479df95-9r42p.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.pD5gJ3NuC8/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.pD5gJ3NuC8/select-1.sql ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cHgikIrIiw +++ mktemp ++ local LAST_ERR=/tmp/tmp.wIhXJR9D1H ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cHgikIrIiw ++ cat /tmp/tmp.wIhXJR9D1H Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.cHgikIrIiw /tmp/tmp.wIhXJR9D1H ++ return 0 + '[' '' ']' + sleep 20 + desc 'add backups schedule for pvc storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for pvc storage ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.NwksJdgl5i +++ mktemp ++ local LAST_ERR=/tmp/tmp.H1NvGq8GJQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NwksJdgl5i ++ cat /tmp/tmp.H1NvGq8GJQ ++ rm /tmp/tmp.NwksJdgl5i /tmp/tmp.H1NvGq8GJQ ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1710-fb797906-3-cluster6 --namespace=scheduled-backup-19247 ++ mktemp + local LAST_OUT=/tmp/tmp.0njrQjwAVd ++ mktemp + local LAST_ERR=/tmp/tmp.ABOIi1OA22 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1710-fb797906-3-cluster6 --namespace=scheduled-backup-19247 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0njrQjwAVd Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1710-fb797906-3-cluster6" modified. + cat /tmp/tmp.ABOIi1OA22 + rm /tmp/tmp.0njrQjwAVd /tmp/tmp.ABOIi1OA22 + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19247~ + local LAST_OUT=/tmp/tmp.tNoFEMhBbT + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.4OyJKznhjP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tNoFEMhBbT perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.4OyJKznhjP + rm /tmp/tmp.tNoFEMhBbT /tmp/tmp.4OyJKznhjP + return 0 + label_node ++ kubectl_bin get nodes --no-headers=true ++ grep -v master ++ head -n1 +++ mktemp ++ awk '{print $1}' ++ local LAST_OUT=/tmp/tmp.lRvXC67fDM +++ mktemp ++ local LAST_ERR=/tmp/tmp.mlTJNjm8u7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get nodes --no-headers=true ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lRvXC67fDM ++ cat /tmp/tmp.mlTJNjm8u7 ++ rm /tmp/tmp.lRvXC67fDM /tmp/tmp.mlTJNjm8u7 ++ return 0 + LABELED_NODE=gke-jen-pxc-1710-fb79790-default-pool-9e0ffb57-gg6v + kubectl_bin label nodes gke-jen-pxc-1710-fb79790-default-pool-9e0ffb57-gg6v backupWorker=True --overwrite ++ mktemp + local LAST_OUT=/tmp/tmp.yjq1bE47wM ++ mktemp + local LAST_ERR=/tmp/tmp.KbfEOKVWRB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label nodes gke-jen-pxc-1710-fb79790-default-pool-9e0ffb57-gg6v backupWorker=True --overwrite + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yjq1bE47wM node/gke-jen-pxc-1710-fb79790-default-pool-9e0ffb57-gg6v not labeled + cat /tmp/tmp.KbfEOKVWRB + rm /tmp/tmp.yjq1bE47wM /tmp/tmp.KbfEOKVWRB + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_OUT=/tmp/tmp.xjf0XkIQXI + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19247~ + local LAST_ERR=/tmp/tmp.YX9RcP5OJW + local exit_status=0 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ seq 0 2 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xjf0XkIQXI perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.YX9RcP5OJW + rm /tmp/tmp.xjf0XkIQXI /tmp/tmp.YX9RcP5OJW + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.EvzmU2Z1dA +++ mktemp ++ local LAST_ERR=/tmp/tmp.9nWhgsKzbs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EvzmU2Z1dA ++ cat /tmp/tmp.9nWhgsKzbs ++ rm /tmp/tmp.EvzmU2Z1dA /tmp/tmp.9nWhgsKzbs ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.ty1O6qOGge +++ mktemp ++ local LAST_ERR=/tmp/tmp.CZwd9jGfHI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ty1O6qOGge ++ cat /tmp/tmp.CZwd9jGfHI ++ rm /tmp/tmp.ty1O6qOGge /tmp/tmp.CZwd9jGfHI ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.VPiLyltBnS ++ mktemp + local LAST_ERR=/tmp/tmp.Jh2pcARkdG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VPiLyltBnS NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-2024514173439-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2024514173439-q6fav Running 15s + cat /tmp/tmp.Jh2pcARkdG + rm /tmp/tmp.VPiLyltBnS /tmp/tmp.Jh2pcARkdG + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.BBJe4xJp5E ++ mktemp + local LAST_ERR=/tmp/tmp.9T0PI4K8xD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BBJe4xJp5E NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-wgdxc 1/1 Running 0 7m11s pxc-client-64b479df95-9r42p 1/1 Running 0 5m58s scheduled-backup-proxysql-0 3/3 Running 0 5m56s scheduled-backup-proxysql-1 3/3 Running 0 5m32s scheduled-backup-pxc-0 1/1 Running 0 5m56s scheduled-backup-pxc-1 1/1 Running 0 5m14s scheduled-backup-pxc-2 1/1 Running 0 4m1s xb-cron-scheduled-backup-pvc-2024514173439-q6fav-hmlnl 1/1 Running 0 15s + cat /tmp/tmp.9T0PI4K8xD + rm /tmp/tmp.BBJe4xJp5E /tmp/tmp.9T0PI4K8xD + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep -vE ':Succeeded|:Failed' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pguvzmp2CD +++ mktemp ++ local LAST_ERR=/tmp/tmp.6YLqLE31dt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Pguvzmp2CD ++ cat /tmp/tmp.6YLqLE31dt ++ rm /tmp/tmp.Pguvzmp2CD /tmp/tmp.6YLqLE31dt ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.yI7UbiJyoj +++ mktemp ++ local LAST_ERR=/tmp/tmp.fwkhK0zqB8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yI7UbiJyoj ++ cat /tmp/tmp.fwkhK0zqB8 ++ rm /tmp/tmp.yI7UbiJyoj /tmp/tmp.fwkhK0zqB8 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.Q7w6N7qGnh ++ mktemp + local LAST_ERR=/tmp/tmp.JXJE3G5rVJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Q7w6N7qGnh NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-2024514173439-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2024514173439-q6fav Succeeded 7s 45s + cat /tmp/tmp.JXJE3G5rVJ + rm /tmp/tmp.Q7w6N7qGnh /tmp/tmp.JXJE3G5rVJ + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.P8qEFcPQ7P ++ mktemp + local LAST_ERR=/tmp/tmp.Z4kAi1VCsu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.P8qEFcPQ7P NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-wgdxc 1/1 Running 0 7m42s pxc-client-64b479df95-9r42p 1/1 Running 0 6m29s scheduled-backup-proxysql-0 3/3 Running 0 6m27s scheduled-backup-proxysql-1 3/3 Running 0 6m3s scheduled-backup-pxc-0 1/1 Running 0 6m27s scheduled-backup-pxc-1 1/1 Running 0 5m45s scheduled-backup-pxc-2 1/1 Running 0 4m32s xb-cron-scheduled-backup-pvc-2024514173439-q6fav-hmlnl 0/1 Completed 0 46s + cat /tmp/tmp.Z4kAi1VCsu + rm /tmp/tmp.P8qEFcPQ7P /tmp/tmp.Z4kAi1VCsu + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.HxmrqVjmbP +++ mktemp ++ local LAST_ERR=/tmp/tmp.fIkqIKi6PB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HxmrqVjmbP ++ cat /tmp/tmp.fIkqIKi6PB ++ rm /tmp/tmp.HxmrqVjmbP /tmp/tmp.fIkqIKi6PB ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.lYlGMy4jgh +++ mktemp ++ local LAST_ERR=/tmp/tmp.DwHbuUYZpd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lYlGMy4jgh ++ cat /tmp/tmp.DwHbuUYZpd ++ rm /tmp/tmp.lYlGMy4jgh /tmp/tmp.DwHbuUYZpd ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.KYLiAGrr5L + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19247~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_ERR=/tmp/tmp.E8fXPpv65W + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KYLiAGrr5L perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.E8fXPpv65W + rm /tmp/tmp.KYLiAGrr5L /tmp/tmp.E8fXPpv65W + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.6qg6hGtDxh + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + local LAST_ERR=/tmp/tmp.THGp8gKGea + local exit_status=0 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19247~ + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6qg6hGtDxh perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.THGp8gKGea + rm /tmp/tmp.6qg6hGtDxh /tmp/tmp.THGp8gKGea + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.gnhuPpbyiV +++ mktemp ++ local LAST_ERR=/tmp/tmp.EOzLvkZQwE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gnhuPpbyiV ++ cat /tmp/tmp.EOzLvkZQwE ++ rm /tmp/tmp.gnhuPpbyiV /tmp/tmp.EOzLvkZQwE ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.lLFuipCn6s +++ mktemp ++ local LAST_ERR=/tmp/tmp.gdzPBYG67A ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lLFuipCn6s ++ cat /tmp/tmp.gdzPBYG67A ++ rm /tmp/tmp.lLFuipCn6s /tmp/tmp.gdzPBYG67A ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.9YCdSq500n ++ mktemp + local LAST_ERR=/tmp/tmp.usuT03XPiX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9YCdSq500n NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2024514173647-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2024-05-14-17:36:47-full Running 11s cron-scheduled-backup-pvc-2024514173439-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2024514173439-q6fav Succeeded 101s 2m19s + cat /tmp/tmp.usuT03XPiX + rm /tmp/tmp.9YCdSq500n /tmp/tmp.usuT03XPiX + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.TOMKQQiCGK ++ mktemp + local LAST_ERR=/tmp/tmp.F9xKjMBQT9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TOMKQQiCGK NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-wgdxc 1/1 Running 0 9m15s pxc-client-64b479df95-9r42p 1/1 Running 0 8m2s scheduled-backup-proxysql-0 3/3 Running 0 8m scheduled-backup-proxysql-1 3/3 Running 0 7m36s scheduled-backup-pxc-0 1/1 Running 0 8m scheduled-backup-pxc-1 1/1 Running 0 7m18s scheduled-backup-pxc-2 1/1 Running 0 6m5s xb-cron-scheduled-backup-aws-s3-2024514173647-q6fav-lh7jt 1/1 Running 0 11s xb-cron-scheduled-backup-pvc-2024514173439-q6fav-hmlnl 0/1 Completed 0 2m19s + cat /tmp/tmp.F9xKjMBQT9 + rm /tmp/tmp.TOMKQQiCGK /tmp/tmp.F9xKjMBQT9 + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.rid2hb9VjI +++ mktemp ++ local LAST_ERR=/tmp/tmp.kj3IW6VL3p ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rid2hb9VjI ++ cat /tmp/tmp.kj3IW6VL3p ++ rm /tmp/tmp.rid2hb9VjI /tmp/tmp.kj3IW6VL3p ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.L02aEx2dnf +++ mktemp ++ local LAST_ERR=/tmp/tmp.qaPAEa2HnG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.L02aEx2dnf ++ cat /tmp/tmp.qaPAEa2HnG ++ rm /tmp/tmp.L02aEx2dnf /tmp/tmp.qaPAEa2HnG ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.uysAcY9yjq ++ mktemp + local LAST_ERR=/tmp/tmp.rRd4IhJXya + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uysAcY9yjq NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2024514173647-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2024-05-14-17:36:47-full Succeeded 3s 40s cron-scheduled-backup-pvc-2024514173439-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2024514173439-q6fav Succeeded 2m10s 2m48s + cat /tmp/tmp.rRd4IhJXya + rm /tmp/tmp.uysAcY9yjq /tmp/tmp.rRd4IhJXya + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.LlQYdj3alz ++ mktemp + local LAST_ERR=/tmp/tmp.yqZ9M0Sy6u + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LlQYdj3alz NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-wgdxc 1/1 Running 0 9m45s pxc-client-64b479df95-9r42p 1/1 Running 0 8m32s scheduled-backup-proxysql-0 3/3 Running 0 8m30s scheduled-backup-proxysql-1 3/3 Running 0 8m6s scheduled-backup-pxc-0 1/1 Running 0 8m30s scheduled-backup-pxc-1 1/1 Running 0 7m48s scheduled-backup-pxc-2 1/1 Running 0 6m35s xb-cron-scheduled-backup-aws-s3-2024514173647-q6fav-lh7jt 0/1 Completed 0 41s xb-cron-scheduled-backup-pvc-2024514173439-q6fav-hmlnl 0/1 Completed 0 2m49s + cat /tmp/tmp.yqZ9M0Sy6u + rm /tmp/tmp.LlQYdj3alz /tmp/tmp.yqZ9M0Sy6u + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.IZXBXIy1FQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.DedOtR8KXp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IZXBXIy1FQ ++ cat /tmp/tmp.DedOtR8KXp ++ rm /tmp/tmp.IZXBXIy1FQ /tmp/tmp.DedOtR8KXp ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.xQ6yufzQTV +++ mktemp ++ local LAST_ERR=/tmp/tmp.fo8JTBXnj8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xQ6yufzQTV ++ cat /tmp/tmp.fo8JTBXnj8 ++ rm /tmp/tmp.xQ6yufzQTV /tmp/tmp.fo8JTBXnj8 ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for minio storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for minio storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.XgklTl1qpF + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19247~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.V71aTRhmu8 + local exit_status=0 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XgklTl1qpF perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.V71aTRhmu8 + rm /tmp/tmp.XgklTl1qpF /tmp/tmp.V71aTRhmu8 + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.MbFHkrXVgp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19247~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.W7peH3KuFz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MbFHkrXVgp perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.W7peH3KuFz + rm /tmp/tmp.MbFHkrXVgp /tmp/tmp.W7peH3KuFz + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.A2v5vBDhWg +++ mktemp ++ local LAST_ERR=/tmp/tmp.6U8a1q1g2x ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A2v5vBDhWg ++ cat /tmp/tmp.6U8a1q1g2x ++ rm /tmp/tmp.A2v5vBDhWg /tmp/tmp.6U8a1q1g2x ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.gX1VEmWW2c +++ mktemp ++ local LAST_ERR=/tmp/tmp.mqqWjmkkwX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gX1VEmWW2c ++ cat /tmp/tmp.mqqWjmkkwX ++ rm /tmp/tmp.gX1VEmWW2c /tmp/tmp.mqqWjmkkwX ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.YOJBenhhd6 ++ mktemp + local LAST_ERR=/tmp/tmp.SImfEwVnWO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YOJBenhhd6 NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2024514173647-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2024-05-14-17:36:47-full Succeeded 101s 2m18s cron-scheduled-backup-minio-2024514173851-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2024-05-14-17:38:51-full Running 14s cron-scheduled-backup-pvc-2024514173439-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2024514173439-q6fav Succeeded 3m48s 4m26s + cat /tmp/tmp.SImfEwVnWO + rm /tmp/tmp.YOJBenhhd6 /tmp/tmp.SImfEwVnWO + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.yNVUn1q0Xt ++ mktemp + local LAST_ERR=/tmp/tmp.3kTmA93xV7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yNVUn1q0Xt NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-wgdxc 1/1 Running 0 11m pxc-client-64b479df95-9r42p 1/1 Running 0 10m scheduled-backup-proxysql-0 3/3 Running 0 10m scheduled-backup-proxysql-1 3/3 Running 0 9m43s scheduled-backup-pxc-0 1/1 Running 0 10m scheduled-backup-pxc-1 1/1 Running 0 9m25s scheduled-backup-pxc-2 1/1 Running 0 8m12s xb-cron-scheduled-backup-aws-s3-2024514173647-q6fav-lh7jt 0/1 Completed 0 2m18s xb-cron-scheduled-backup-minio-2024514173851-q6fav-46cz5 1/1 Running 0 14s xb-cron-scheduled-backup-pvc-2024514173439-q6fav-hmlnl 0/1 Completed 0 4m26s + cat /tmp/tmp.3kTmA93xV7 + rm /tmp/tmp.yNVUn1q0Xt /tmp/tmp.3kTmA93xV7 + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.rzj2AVt6JL +++ mktemp ++ local LAST_ERR=/tmp/tmp.CszgRRdWNr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rzj2AVt6JL ++ cat /tmp/tmp.CszgRRdWNr ++ rm /tmp/tmp.rzj2AVt6JL /tmp/tmp.CszgRRdWNr ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.1Nw2NnvLch +++ mktemp ++ local LAST_ERR=/tmp/tmp.LYR9I617BH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1Nw2NnvLch ++ cat /tmp/tmp.LYR9I617BH ++ rm /tmp/tmp.1Nw2NnvLch /tmp/tmp.LYR9I617BH ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for gcs storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for gcs storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.twBpkHNLdl + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19247~ + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.NAKQqsMZ0O + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.twBpkHNLdl perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.NAKQqsMZ0O + rm /tmp/tmp.twBpkHNLdl /tmp/tmp.NAKQqsMZ0O + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906#' + local LAST_OUT=/tmp/tmp.dm1owm5tmV + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_ERR=/tmp/tmp.1dg5qd3uxJ + local exit_status=0 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19247~ ++ seq 0 2 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dm1owm5tmV perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.1dg5qd3uxJ + rm /tmp/tmp.dm1owm5tmV /tmp/tmp.1dg5qd3uxJ + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.14CceJbIUW +++ mktemp ++ local LAST_ERR=/tmp/tmp.sA8lKzzNAW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.14CceJbIUW ++ cat /tmp/tmp.sA8lKzzNAW ++ rm /tmp/tmp.14CceJbIUW /tmp/tmp.sA8lKzzNAW ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.fBr3NEFxLw +++ mktemp ++ local LAST_ERR=/tmp/tmp.aAKjR0jpZ3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fBr3NEFxLw ++ cat /tmp/tmp.aAKjR0jpZ3 ++ rm /tmp/tmp.fBr3NEFxLw /tmp/tmp.aAKjR0jpZ3 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.TaSqPmAtbT ++ mktemp + local LAST_ERR=/tmp/tmp.paHTh2SXAO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TaSqPmAtbT NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2024514173647-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2024-05-14-17:36:47-full Succeeded 3m20s 3m57s cron-scheduled-backup-gcp-cs-2024514174029-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2024-05-14-17:40:29-full Running 15s cron-scheduled-backup-minio-2024514173851-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2024-05-14-17:38:51-full Succeeded 87s 113s cron-scheduled-backup-pvc-2024514173439-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2024514173439-q6fav Succeeded 5m27s 6m5s + cat /tmp/tmp.paHTh2SXAO + rm /tmp/tmp.TaSqPmAtbT /tmp/tmp.paHTh2SXAO + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.GQ1ryOeR1l ++ mktemp + local LAST_ERR=/tmp/tmp.y4pGDDrD42 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GQ1ryOeR1l NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-wgdxc 1/1 Running 0 13m pxc-client-64b479df95-9r42p 1/1 Running 0 11m scheduled-backup-proxysql-0 3/3 Running 0 11m scheduled-backup-proxysql-1 3/3 Running 0 11m scheduled-backup-pxc-0 1/1 Running 0 11m scheduled-backup-pxc-1 1/1 Running 0 11m scheduled-backup-pxc-2 1/1 Running 0 9m52s xb-cron-scheduled-backup-aws-s3-2024514173647-q6fav-lh7jt 0/1 Completed 0 3m58s xb-cron-scheduled-backup-gcp-cs-2024514174029-q6fav-g8qlp 1/1 Running 0 16s xb-cron-scheduled-backup-minio-2024514173851-q6fav-46cz5 0/1 Completed 0 114s xb-cron-scheduled-backup-pvc-2024514173439-q6fav-hmlnl 0/1 Completed 0 6m6s + cat /tmp/tmp.y4pGDDrD42 + rm /tmp/tmp.GQ1ryOeR1l /tmp/tmp.y4pGDDrD42 + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.onLM0qMS2V +++ mktemp ++ local LAST_ERR=/tmp/tmp.JXOzK6iwcv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.onLM0qMS2V ++ cat /tmp/tmp.JXOzK6iwcv ++ rm /tmp/tmp.onLM0qMS2V /tmp/tmp.JXOzK6iwcv ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.cIaeNZDyiy +++ mktemp ++ local LAST_ERR=/tmp/tmp.VompAvJ5M9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cIaeNZDyiy ++ cat /tmp/tmp.VompAvJ5M9 ++ rm /tmp/tmp.cIaeNZDyiy /tmp/tmp.VompAvJ5M9 ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for azure storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for azure storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-azure.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + local LAST_OUT=/tmp/tmp.S4z9s2tEE1 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19247~ ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_ERR=/tmp/tmp.ZJXBZSCc5L + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#apply:.*#apply: Never#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.S4z9s2tEE1 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.ZJXBZSCc5L + rm /tmp/tmp.S4z9s2tEE1 /tmp/tmp.ZJXBZSCc5L + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1710/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906#' + local LAST_OUT=/tmp/tmp.y6phC6cC4v + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + local LAST_ERR=/tmp/tmp.lmSHYIILiz + local exit_status=0 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19247~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.y6phC6cC4v perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.lmSHYIILiz + rm /tmp/tmp.y6phC6cC4v /tmp/tmp.lmSHYIILiz + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.X7k9H7wzEX +++ mktemp ++ local LAST_ERR=/tmp/tmp.6n8HGS5Q9g ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.X7k9H7wzEX ++ cat /tmp/tmp.6n8HGS5Q9g ++ rm /tmp/tmp.X7k9H7wzEX /tmp/tmp.6n8HGS5Q9g ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.3OpGA19vcz +++ mktemp ++ local LAST_ERR=/tmp/tmp.ugvByQHKMH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3OpGA19vcz ++ cat /tmp/tmp.ugvByQHKMH ++ rm /tmp/tmp.3OpGA19vcz /tmp/tmp.ugvByQHKMH ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.pjSI4R9F29 ++ mktemp + local LAST_ERR=/tmp/tmp.kBBpK3vyrd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pjSI4R9F29 NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2024514173647-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2024-05-14-17:36:47-full Succeeded 4m56s 5m33s cron-scheduled-backup-azure-blob-202451417429-q6fav scheduled-backup azure-blob azure://operator-testing/scheduled-backup-2024-05-14-17:42:09-full Running 11s cron-scheduled-backup-gcp-cs-2024514174029-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2024-05-14-17:40:29-full Succeeded 80s 111s cron-scheduled-backup-minio-2024514173851-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2024-05-14-17:38:51-full Succeeded 3m3s 3m29s cron-scheduled-backup-pvc-2024514173439-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2024514173439-q6fav Succeeded 7m3s 7m41s + cat /tmp/tmp.kBBpK3vyrd + rm /tmp/tmp.pjSI4R9F29 /tmp/tmp.kBBpK3vyrd + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.K4rVOX1QYK ++ mktemp + local LAST_ERR=/tmp/tmp.xbdXqj3P5x + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.K4rVOX1QYK NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-wgdxc 1/1 Running 0 14m pxc-client-64b479df95-9r42p 1/1 Running 0 13m scheduled-backup-proxysql-0 3/3 Running 0 13m scheduled-backup-proxysql-1 3/3 Running 0 12m scheduled-backup-pxc-0 1/1 Running 0 13m scheduled-backup-pxc-1 1/1 Running 0 12m scheduled-backup-pxc-2 1/1 Running 0 11m xb-cron-scheduled-backup-aws-s3-2024514173647-q6fav-lh7jt 0/1 Completed 0 5m33s xb-cron-scheduled-backup-azure-blob-202451417429-q6fav-fpt9k 1/1 Running 0 11s xb-cron-scheduled-backup-gcp-cs-2024514174029-q6fav-g8qlp 0/1 Completed 0 111s xb-cron-scheduled-backup-minio-2024514173851-q6fav-46cz5 0/1 Completed 0 3m29s xb-cron-scheduled-backup-pvc-2024514173439-q6fav-hmlnl 0/1 Completed 0 7m41s + cat /tmp/tmp.xbdXqj3P5x + rm /tmp/tmp.K4rVOX1QYK /tmp/tmp.xbdXqj3P5x + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.C0FVBysALM +++ mktemp ++ local LAST_ERR=/tmp/tmp.3nhYnxTQJS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.C0FVBysALM ++ cat /tmp/tmp.3nhYnxTQJS ++ rm /tmp/tmp.C0FVBysALM /tmp/tmp.3nhYnxTQJS ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.rZUnYG16EJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.iYpqOWo9Ir ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rZUnYG16EJ ++ cat /tmp/tmp.iYpqOWo9Ir ++ rm /tmp/tmp.rZUnYG16EJ /tmp/tmp.iYpqOWo9Ir ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.GXYlsdMzxC ++ mktemp + local LAST_ERR=/tmp/tmp.C0vfvUbXxu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GXYlsdMzxC NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-2024514173647-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2024-05-14-17:36:47-full Succeeded 5m25s 6m2s cron-scheduled-backup-azure-blob-202451417429-q6fav scheduled-backup azure-blob azure://operator-testing/scheduled-backup-2024-05-14-17:42:09-full Running 40s cron-scheduled-backup-gcp-cs-2024514174029-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2024-05-14-17:40:29-full Succeeded 109s 2m20s cron-scheduled-backup-minio-2024514173851-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2024-05-14-17:38:51-full Succeeded 3m32s 3m58s cron-scheduled-backup-pvc-2024514173439-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-2024514173439-q6fav Succeeded 7m32s 8m10s + cat /tmp/tmp.C0vfvUbXxu + rm /tmp/tmp.GXYlsdMzxC /tmp/tmp.C0vfvUbXxu + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.GTXsXiY0vJ ++ mktemp + local LAST_ERR=/tmp/tmp.5B1Nb3RaLe + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GTXsXiY0vJ NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-tmcb7 0/1 ContainerCreating 0 13s minio-service-76ffcfd45-wgdxc 0/1 Completed 0 15m pxc-client-64b479df95-9r42p 1/1 Running 0 13m scheduled-backup-proxysql-0 0/3 Init:0/1 0 13s scheduled-backup-proxysql-1 3/3 Running 0 13m scheduled-backup-pxc-0 1/1 Running 0 13m scheduled-backup-pxc-1 1/1 Running 0 13m scheduled-backup-pxc-2 1/1 Running 0 11m xb-cron-scheduled-backup-aws-s3-2024514173647-q6fav-lh7jt 0/1 Completed 0 6m2s xb-cron-scheduled-backup-azure-blob-202451417429-q6fav-fpt9k 0/1 Completed 0 40s xb-cron-scheduled-backup-gcp-cs-2024514174029-q6fav-g8qlp 0/1 Completed 0 2m20s xb-cron-scheduled-backup-minio-2024514173851-q6fav-46cz5 0/1 Completed 0 3m58s xb-cron-scheduled-backup-pvc-2024514173439-q6fav-hmlnl 0/1 Completed 0 8m10s + cat /tmp/tmp.5B1Nb3RaLe + rm /tmp/tmp.GTXsXiY0vJ /tmp/tmp.5B1Nb3RaLe + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.ONoOY3qBpx +++ mktemp ++ local LAST_ERR=/tmp/tmp.xm1tMT5a0W ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ONoOY3qBpx ++ cat /tmp/tmp.xm1tMT5a0W ++ rm /tmp/tmp.ONoOY3qBpx /tmp/tmp.xm1tMT5a0W ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.RxWc9CIGnS +++ mktemp ++ local LAST_ERR=/tmp/tmp.R24DuoWMfV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RxWc9CIGnS ++ cat /tmp/tmp.R24DuoWMfV ++ rm /tmp/tmp.RxWc9CIGnS /tmp/tmp.R24DuoWMfV ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 scheduled-backup-pxc-0.Name: scheduled-backup-pxc-0 Namespace: scheduled-backup-19247 Priority: 0 Service Account: default Node: Labels: app.kubernetes.io/component=pxc app.kubernetes.io/instance=scheduled-backup app.kubernetes.io/managed-by=percona-xtradb-cluster-operator app.kubernetes.io/name=percona-xtradb-cluster app.kubernetes.io/part-of=percona-xtradb-cluster controller-revision-hash=scheduled-backup-pxc-8964f957c statefulset.kubernetes.io/pod-name=scheduled-backup-pxc-0 Annotations: cloud.google.com/cluster_autoscaler_unhelpable_since: 2024-05-14T17:42:52+0000 cloud.google.com/cluster_autoscaler_unhelpable_until: Inf percona.com/configuration-hash: d41d8cd98f00b204e9800998ecf8427e percona.com/ssl-hash: cfd6a52398268173b51d7cdb1331c09a percona.com/ssl-internal-hash: 751e59a1c1d94ac67f9d6bb6e756acdc Status: Pending IP: IPs: Controlled By: StatefulSet/scheduled-backup-pxc Init Containers: pxc-init: Image: perconalab/percona-xtradb-cluster-operator:PR-1710-fb797906 Port: Host Port: Command: /pxc-init-entrypoint.sh Limits: cpu: 50m memory: 50M Requests: cpu: 50m memory: 50M Environment: Mounts: /var/lib/mysql from datadir (rw) /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-7njwx (ro) Containers: pxc: Image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7 Ports: 3306/TCP, 4444/TCP, 4567/TCP, 4568/TCP, 33062/TCP, 33060/TCP Host Ports: 0/TCP, 0/TCP, 0/TCP, 0/TCP, 0/TCP, 0/TCP Command: /var/lib/mysql/pxc-entrypoint.sh Args: mysqld Liveness: exec [/var/lib/mysql/liveness-check.sh] delay=300s timeout=5s period=10s #success=1 #failure=3 Readiness: exec [/var/lib/mysql/readiness-check.sh] delay=15s timeout=15s period=30s #success=1 #failure=5 Environment Variables from: scheduled-backup-env-vars-pxc Secret Optional: true Environment: PXC_SERVICE: scheduled-backup-pxc-unready MONITOR_HOST: % MYSQL_ROOT_PASSWORD: Optional: false XTRABACKUP_PASSWORD: Optional: false MONITOR_PASSWORD: Optional: false CLUSTER_HASH: 3571499 OPERATOR_ADMIN_PASSWORD: Optional: false LIVENESS_CHECK_TIMEOUT: 5 READINESS_CHECK_TIMEOUT: 15 DEFAULT_AUTHENTICATION_PLUGIN: mysql_native_password Mounts: /etc/my.cnf.d from auto-config (rw) /etc/mysql/init-file from mysql-init-file (rw) /etc/mysql/mysql-users-secret from mysql-users-secret-file (rw) /etc/mysql/ssl from ssl (rw) /etc/mysql/ssl-internal from ssl-internal (rw) /etc/mysql/vault-keyring-secret from vault-keyring-secret (rw) /etc/percona-xtradb-cluster.conf.d from config (rw) /tmp from tmp (rw) /var/lib/mysql from datadir (rw) /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-7njwx (ro) Conditions: Type Status PodScheduled False Volumes: datadir: Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace) ClaimName: datadir-scheduled-backup-pxc-0 ReadOnly: false tmp: Type: EmptyDir (a temporary directory that shares a pod's lifetime) Medium: SizeLimit: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: scheduled-backup-pxc Optional: true ssl-internal: Type: Secret (a volume populated by a Secret) SecretName: some-name-ssl-internal Optional: true ssl: Type: Secret (a volume populated by a Secret) SecretName: some-name-ssl Optional: false auto-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: auto-scheduled-backup-pxc Optional: true vault-keyring-secret: Type: Secret (a volume populated by a Secret) SecretName: scheduled-backup-vault Optional: true mysql-users-secret-file: Type: Secret (a volume populated by a Secret) SecretName: internal-scheduled-backup Optional: false mysql-init-file: Type: Secret (a volume populated by a Secret) SecretName: scheduled-backup-mysql-init Optional: true kube-api-access-7njwx: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedScheduling 26s default-scheduler 0/3 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }, 2 node(s) didn't match pod anti-affinity rules. preemption: 0/3 nodes are available: 1 Preemption is not helpful for scheduling, 2 No preemption victims found for incoming pod.. Warning FailedScheduling 24s default-scheduler 0/3 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }, 2 node(s) didn't match pod anti-affinity rules. preemption: 0/3 nodes are available: 1 Preemption is not helpful for scheduling, 2 No preemption victims found for incoming pod.. Warning FailedScheduling 22s default-scheduler 0/3 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }, 2 node(s) didn't match pod anti-affinity rules. preemption: 0/3 nodes are available: 1 Preemption is not helpful for scheduling, 2 No preemption victims found for incoming pod.. Normal NotTriggerScaleUp 24s cluster-autoscaler pod didn't trigger scale-up: Defaulted container "pxc" out of: pxc, pxc-init (init) Error from server: Get "https://10.213.0.49:10250/containerLogs/pxc-operator/percona-xtradb-cluster-operator-8448ddf579-7lw5z/percona-xtradb-cluster-operator": dial tcp 10.213.0.49:10250: i/o timeout