Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/logs/scheduled-backup-5-7.log Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 + main + create_infra scheduled-backup-29185 + local ns=scheduled-backup-29185 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n scheduled-backup-15447 scheduled-backup --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.Yi6gRBUJvX ++ mktemp + local LAST_ERR=/tmp/tmp.W2Lh3KmxVl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Yi6gRBUJvX perconaxtradbcluster.pxc.percona.com "scheduled-backup" deleted from scheduled-backup-15447 namespace + cat /tmp/tmp.W2Lh3KmxVl + rm /tmp/tmp.Yi6gRBUJvX /tmp/tmp.W2Lh3KmxVl + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.LOPDmWUQDP ++ mktemp + local LAST_ERR=/tmp/tmp.Fu3EV8Uuwe + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LOPDmWUQDP perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-aws-s3-2025103117261-q6fav" deleted from scheduled-backup-15447 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-azure-blob-20251031172344-q6fav" deleted from scheduled-backup-15447 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-gcp-cs-20251031172130-q6fav" deleted from scheduled-backup-15447 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-minio-20251031171538-q6fav" deleted from scheduled-backup-15447 namespace perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-pvc-20251031171154-q6fav" deleted from scheduled-backup-15447 namespace + cat /tmp/tmp.Fu3EV8Uuwe + rm /tmp/tmp.LOPDmWUQDP /tmp/tmp.Fu3EV8Uuwe + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.4UzJvw8usV ++ mktemp + local LAST_ERR=/tmp/tmp.WUxWNYdDLp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4UzJvw8usV perconaxtradbclusterrestore.pxc.percona.com "pvc-20251031171154-q6fav" deleted from scheduled-backup-15447 namespace + cat /tmp/tmp.WUxWNYdDLp + rm /tmp/tmp.4UzJvw8usV /tmp/tmp.WUxWNYdDLp + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + awk '{print$1}' ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.3URypIljKA ++ mktemp + local LAST_OUT=/tmp/tmp.jmPPhPYrKF + local LAST_ERR=/tmp/tmp.ljfIwM4YAZ + local exit_status=0 ++ seq 0 2 ++ mktemp + for i in '$(seq 0 2)' + set +e + local LAST_ERR=/tmp/tmp.5by9siYp3y + local exit_status=0 + kubectl delete namespace pxc-operator ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jmPPhPYrKF + cat /tmp/tmp.5by9siYp3y + rm /tmp/tmp.jmPPhPYrKF /tmp/tmp.5by9siYp3y + return 0 namespace "scheduled-backup-15447" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3URypIljKA namespace "pxc-operator" deleted + cat /tmp/tmp.ljfIwM4YAZ + rm /tmp/tmp.3URypIljKA /tmp/tmp.ljfIwM4YAZ + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.mBokSkOfBl ++ mktemp + local LAST_ERR=/tmp/tmp.aw0qySFgj9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mBokSkOfBl namespace/pxc-operator created + cat /tmp/tmp.aw0qySFgj9 + rm /tmp/tmp.mBokSkOfBl /tmp/tmp.aw0qySFgj9 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.hAQc9SqDNu +++ mktemp ++ local LAST_ERR=/tmp/tmp.YIeo1AerGn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hAQc9SqDNu ++ cat /tmp/tmp.YIeo1AerGn ++ rm /tmp/tmp.hAQc9SqDNu /tmp/tmp.YIeo1AerGn ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2227-2da0e2db-1-cluster4 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.4tPZb0XZ2o ++ mktemp + local LAST_ERR=/tmp/tmp.r6jgvSaluR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2227-2da0e2db-1-cluster4 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4tPZb0XZ2o Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2227-2da0e2db-1-cluster4" modified. + cat /tmp/tmp.r6jgvSaluR + rm /tmp/tmp.4tPZb0XZ2o /tmp/tmp.r6jgvSaluR + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.PcM3XEm88K ++ mktemp + local LAST_ERR=/tmp/tmp.IOatAWHsdH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PcM3XEm88K customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.IOatAWHsdH + rm /tmp/tmp.PcM3XEm88K /tmp/tmp.IOatAWHsdH + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/deploy/cw-rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.T07SbIzwoQ ++ mktemp + local LAST_ERR=/tmp/tmp.XprHqvkmT4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.T07SbIzwoQ clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.XprHqvkmT4 + rm /tmp/tmp.T07SbIzwoQ /tmp/tmp.XprHqvkmT4 + return 0 + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/deploy/cw-operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.eUc4Wf5PSv ++ mktemp + local LAST_ERR=/tmp/tmp.bJSj34PPYR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2227-2da0e2db^' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eUc4Wf5PSv deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.bJSj34PPYR + rm /tmp/tmp.eUc4Wf5PSv /tmp/tmp.bJSj34PPYR + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.6V3kPsaVPd ++ mktemp + local LAST_ERR=/tmp/tmp.b1JfPr6PbO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6V3kPsaVPd pod/percona-xtradb-cluster-operator-554c4f86-ncnv4 condition met + cat /tmp/tmp.b1JfPr6PbO + rm /tmp/tmp.6V3kPsaVPd /tmp/tmp.b1JfPr6PbO + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.N1xyzNVdet +++ mktemp ++ local LAST_ERR=/tmp/tmp.gbVjX8WZXn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.N1xyzNVdet ++ cat /tmp/tmp.gbVjX8WZXn ++ rm /tmp/tmp.N1xyzNVdet /tmp/tmp.gbVjX8WZXn ++ return 0 + wait_pod percona-xtradb-cluster-operator-554c4f86-ncnv4 480 pxc-operator + local pod=percona-xtradb-cluster-operator-554c4f86-ncnv4 + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-554c4f86-ncnv4 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-554c4f86-ncnv4 condition met waiting for pod/percona-xtradb-cluster-operator-554c4f86-ncnv4 to become Ready.Ok + sleep 3 + create_namespace scheduled-backup-29185 + local namespace=scheduled-backup-29185 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces scheduled-backup-29185' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces scheduled-backup-29185 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace scheduled-backup-29185 + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.WcbmYgsWWd ++ mktemp + local LAST_ERR=/tmp/tmp.geQopTUnwj + local exit_status=0 ++ seq 0 2 + local LAST_OUT=/tmp/tmp.HxZX6fpSut ++ mktemp + for i in '$(seq 0 2)' + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.UfpWDIV2eY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-29185 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-29185 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WcbmYgsWWd + cat /tmp/tmp.geQopTUnwj + rm /tmp/tmp.WcbmYgsWWd /tmp/tmp.geQopTUnwj + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-29185 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.HxZX6fpSut + cat /tmp/tmp.UfpWDIV2eY Error from server (NotFound): namespaces "scheduled-backup-29185" not found + rm /tmp/tmp.HxZX6fpSut /tmp/tmp.UfpWDIV2eY + return 1 + : + wait_for_delete namespace/scheduled-backup-29185 + local res=namespace/scheduled-backup-29185 + echo -n 'waiting for namespace/scheduled-backup-29185 to be deleted' waiting for namespace/scheduled-backup-29185 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "scheduled-backup-29185" not found + desc 'create namespace scheduled-backup-29185' + set +o xtrace ----------------------------------------------------------------------------------- create namespace scheduled-backup-29185 ----------------------------------------------------------------------------------- + kubectl_bin create namespace scheduled-backup-29185 ++ mktemp + local LAST_OUT=/tmp/tmp.pbdZPURUvi ++ mktemp + local LAST_ERR=/tmp/tmp.Jpr5CRUzuf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace scheduled-backup-29185 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pbdZPURUvi namespace/scheduled-backup-29185 created + cat /tmp/tmp.Jpr5CRUzuf + rm /tmp/tmp.pbdZPURUvi /tmp/tmp.Jpr5CRUzuf + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.TobYnme0rS +++ mktemp ++ local LAST_ERR=/tmp/tmp.bd2F6C1cYK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TobYnme0rS ++ cat /tmp/tmp.bd2F6C1cYK ++ rm /tmp/tmp.TobYnme0rS /tmp/tmp.bd2F6C1cYK ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2227-2da0e2db-1-cluster4 --namespace=scheduled-backup-29185 ++ mktemp + local LAST_OUT=/tmp/tmp.1QvbUCw6XK ++ mktemp + local LAST_ERR=/tmp/tmp.0bNOthtcDQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2227-2da0e2db-1-cluster4 --namespace=scheduled-backup-29185 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1QvbUCw6XK Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2227-2da0e2db-1-cluster4" modified. + cat /tmp/tmp.0bNOthtcDQ + rm /tmp/tmp.1QvbUCw6XK /tmp/tmp.0bNOthtcDQ + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.cDlFLikEvh ++ mktemp + local LAST_ERR=/tmp/tmp.mAp7xMKxcD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cDlFLikEvh secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.mAp7xMKxcD + rm /tmp/tmp.cDlFLikEvh /tmp/tmp.mAp7xMKxcD + return 0 + start_minio + deploy_helm scheduled-backup-29185 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Fri Oct 31 17:32:44 2025 NAMESPACE: scheduled-backup-29185 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.scheduled-backup-29185.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-29185 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-29185 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-29185 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-29185 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z57ubC4mD1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.dLkjStmphB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.z57ubC4mD1 ++ cat /tmp/tmp.dLkjStmphB ++ rm /tmp/tmp.z57ubC4mD1 /tmp/tmp.dLkjStmphB ++ return 0 + MINIO_POD=minio-service-55fcc5d75f-r4t8x + wait_pod minio-service-55fcc5d75f-r4t8x + local pod=minio-service-55fcc5d75f-r4t8x + local max_retry=480 + local ns= ++ echo minio-service-55fcc5d75f-r4t8x ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minio-service-55fcc5d75f-r4t8x condition met waiting for pod/minio-service-55fcc5d75f-r4t8x to become Ready.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.JHJXxJFbtl ++ mktemp + local LAST_ERR=/tmp/tmp.99H2HSvsMz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JHJXxJFbtl make_bucket: operator-testing pod "aws-cli" deleted from scheduled-backup-29185 namespace + cat /tmp/tmp.99H2HSvsMz All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_scheduled-backup-29185 + rm /tmp/tmp.JHJXxJFbtl /tmp/tmp.99H2HSvsMz + return 0 + cluster=scheduled-backup + cat - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.OlzSNPwRmr ++ mktemp + local LAST_ERR=/tmp/tmp.1Lun26IOPb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OlzSNPwRmr priorityclass.scheduling.k8s.io/high-priority configured + cat /tmp/tmp.1Lun26IOPb + rm /tmp/tmp.OlzSNPwRmr /tmp/tmp.1Lun26IOPb + return 0 + spinup_pxc scheduled-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local cluster=scheduled-backup + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.PA9SdenKO7 ++ mktemp + local LAST_ERR=/tmp/tmp.TD2jgln5r0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PA9SdenKO7 secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.TD2jgln5r0 + rm /tmp/tmp.PA9SdenKO7 /tmp/tmp.TD2jgln5r0 + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/conf/client.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29185~ + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2227-2da0e2db#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/conf/client.yml + local LAST_OUT=/tmp/tmp.Ne6jYA6uoT ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_ERR=/tmp/tmp.tLKSrSf6VD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Ne6jYA6uoT deployment.apps/pxc-client created + cat /tmp/tmp.tLKSrSf6VD + rm /tmp/tmp.Ne6jYA6uoT /tmp/tmp.tLKSrSf6VD + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p ]] + [[ scheduled-backup == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.FPEeMoqVYC + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2227-2da0e2db#' ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + local LAST_ERR=/tmp/tmp.3E64blEqp1 + local exit_status=0 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29185~ ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FPEeMoqVYC perconaxtradbcluster.pxc.percona.com/scheduled-backup created + cat /tmp/tmp.3E64blEqp1 + rm /tmp/tmp.FPEeMoqVYC /tmp/tmp.3E64blEqp1 + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy scheduled-backup ++ local target_cluster=scheduled-backup +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.N01FQxeXHW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qwVfnEu2BW +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.N01FQxeXHW +++ cat /tmp/tmp.qwVfnEu2BW +++ rm /tmp/tmp.N01FQxeXHW /tmp/tmp.qwVfnEu2BW +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Td6suSNjvj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3fqFPT2NpI +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Td6suSNjvj +++ cat /tmp/tmp.3fqFPT2NpI +++ rm /tmp/tmp.Td6suSNjvj /tmp/tmp.3fqFPT2NpI +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scheduled-backup-proxysql ++ return + local proxy=scheduled-backup-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-29185 ++ mktemp + local LAST_OUT=/tmp/tmp.JaGgQV1ufS ++ mktemp + local LAST_ERR=/tmp/tmp.kDb1dUY7g9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-29185 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-29185 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-29185 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.JaGgQV1ufS + cat /tmp/tmp.kDb1dUY7g9 error: no matching resources found + rm /tmp/tmp.JaGgQV1ufS /tmp/tmp.kDb1dUY7g9 + return 1 + true + wait_for_running scheduled-backup-proxysql 1 + local name=scheduled-backup-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-proxysql-0 480 + local pod=scheduled-backup-proxysql-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-proxysql-0 + local container=proxysql + set +o xtrace pod/scheduled-backup-proxysql-0 condition met waiting for pod/scheduled-backup-proxysql-0 to become Ready.Ok + wait_for_running scheduled-backup-pxc 3 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 480 + local pod=scheduled-backup-pxc-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo scheduled-backup-pxc-0 + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 480 + local pod=scheduled-backup-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-1 ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 480 + local pod=scheduled-backup-pxc-2 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.c2K9Su5bCq +++ mktemp ++ local LAST_ERR=/tmp/tmp.kRI4Ai20mk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.c2K9Su5bCq ++ cat /tmp/tmp.kRI4Ai20mk ++ rm /tmp/tmp.c2K9Su5bCq /tmp/tmp.kRI4Ai20mk ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xokurCP2iC +++ mktemp ++ local LAST_ERR=/tmp/tmp.xC6kt2qDnL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xokurCP2iC ++ cat /tmp/tmp.xC6kt2qDnL Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.xokurCP2iC /tmp/tmp.xC6kt2qDnL ++ return 0 + [[ -n '' ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UVpTfWvjUI +++ mktemp ++ local LAST_ERR=/tmp/tmp.6zHB5lkay3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UVpTfWvjUI ++ cat /tmp/tmp.6zHB5lkay3 ++ rm /tmp/tmp.UVpTfWvjUI /tmp/tmp.6zHB5lkay3 ++ return 0 + client_pod=pxc-client-857d976497-lvd7p + wait_pod pxc-client-857d976497-lvd7p + local pod=pxc-client-857d976497-lvd7p + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-857d976497-lvd7p + local container= + set +o xtrace pod/pxc-client-857d976497-lvd7p condition met waiting for pod/pxc-client-857d976497-lvd7p to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h scheduled-backup-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xYu58wlpGS +++ mktemp ++ local LAST_ERR=/tmp/tmp.TCESYO6UCK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xYu58wlpGS ++ cat /tmp/tmp.TCESYO6UCK ++ rm /tmp/tmp.xYu58wlpGS /tmp/tmp.TCESYO6UCK ++ return 0 + client_pod=pxc-client-857d976497-lvd7p + wait_pod pxc-client-857d976497-lvd7p + local pod=pxc-client-857d976497-lvd7p + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-lvd7p ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-857d976497-lvd7p condition met waiting for pod/pxc-client-857d976497-lvd7p to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z2KWO3eijc +++ mktemp ++ local LAST_ERR=/tmp/tmp.gBUt3DSKJy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Z2KWO3eijc ++ cat /tmp/tmp.gBUt3DSKJy ++ rm /tmp/tmp.Z2KWO3eijc /tmp/tmp.gBUt3DSKJy ++ return 0 + client_pod=pxc-client-857d976497-lvd7p + wait_pod pxc-client-857d976497-lvd7p + local pod=pxc-client-857d976497-lvd7p + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-857d976497-lvd7p + local container= + set +o xtrace pod/pxc-client-857d976497-lvd7p condition met waiting for pod/pxc-client-857d976497-lvd7p to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.oHDGkUIyYT/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.oHDGkUIyYT/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ULxL1fCK21 +++ mktemp ++ local LAST_ERR=/tmp/tmp.uuSHw3EMsS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ULxL1fCK21 ++ cat /tmp/tmp.uuSHw3EMsS ++ rm /tmp/tmp.ULxL1fCK21 /tmp/tmp.uuSHw3EMsS ++ return 0 + client_pod=pxc-client-857d976497-lvd7p + wait_pod pxc-client-857d976497-lvd7p + local pod=pxc-client-857d976497-lvd7p + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-lvd7p ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-857d976497-lvd7p condition met waiting for pod/pxc-client-857d976497-lvd7p to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.oHDGkUIyYT/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.oHDGkUIyYT/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/compare/select-1-57.sql ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SssJKNZLan +++ mktemp ++ local LAST_ERR=/tmp/tmp.p5hRIN8Ud2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SssJKNZLan ++ cat /tmp/tmp.p5hRIN8Ud2 ++ rm /tmp/tmp.SssJKNZLan /tmp/tmp.p5hRIN8Ud2 ++ return 0 + client_pod=pxc-client-857d976497-lvd7p + wait_pod pxc-client-857d976497-lvd7p + local pod=pxc-client-857d976497-lvd7p + local max_retry=480 + local ns= ++ echo pxc-client-857d976497-lvd7p ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-857d976497-lvd7p condition met waiting for pod/pxc-client-857d976497-lvd7p to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.oHDGkUIyYT/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.oHDGkUIyYT/select-1.sql ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4yASL1fE0X +++ mktemp ++ local LAST_ERR=/tmp/tmp.giNqGlZ1Ud ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4yASL1fE0X ++ cat /tmp/tmp.giNqGlZ1Ud Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.4yASL1fE0X /tmp/tmp.giNqGlZ1Ud ++ return 0 + '[' '' ']' + sleep 20 + desc 'add backups schedule for pvc storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for pvc storage ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.aYy0Oxu5wx +++ mktemp ++ local LAST_ERR=/tmp/tmp.LFZPkxoHTW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aYy0Oxu5wx ++ cat /tmp/tmp.LFZPkxoHTW ++ rm /tmp/tmp.aYy0Oxu5wx /tmp/tmp.LFZPkxoHTW ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2227-2da0e2db-1-cluster4 --namespace=scheduled-backup-29185 ++ mktemp + local LAST_OUT=/tmp/tmp.E8iEmh174n ++ mktemp + local LAST_ERR=/tmp/tmp.0D5jK4Ib4i + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2227-2da0e2db-1-cluster4 --namespace=scheduled-backup-29185 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.E8iEmh174n Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2227-2da0e2db-1-cluster4" modified. + cat /tmp/tmp.0D5jK4Ib4i + rm /tmp/tmp.E8iEmh174n /tmp/tmp.0D5jK4Ib4i + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29185~ + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2227-2da0e2db#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + local LAST_OUT=/tmp/tmp.rPI3BFNSQj + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_ERR=/tmp/tmp.Z5YOXYBYey + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rPI3BFNSQj perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.Z5YOXYBYey + rm /tmp/tmp.rPI3BFNSQj /tmp/tmp.Z5YOXYBYey + return 0 + label_node ++ kubectl_bin get nodes --no-headers=true ++ grep -v master ++ head -n1 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.shYhNmX5Bq +++ mktemp ++ local LAST_ERR=/tmp/tmp.YdzQAkWFmm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get nodes --no-headers=true ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.shYhNmX5Bq ++ cat /tmp/tmp.YdzQAkWFmm ++ rm /tmp/tmp.shYhNmX5Bq /tmp/tmp.YdzQAkWFmm ++ return 0 + LABELED_NODE=gke-jen-pxc-2227-2da0e2d-default-pool-455a2d45-drbw + kubectl_bin label nodes gke-jen-pxc-2227-2da0e2d-default-pool-455a2d45-drbw backupWorker=True --overwrite ++ mktemp + local LAST_OUT=/tmp/tmp.UTcMHZWDrz ++ mktemp + local LAST_ERR=/tmp/tmp.kXysJd8eoX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label nodes gke-jen-pxc-2227-2da0e2d-default-pool-455a2d45-drbw backupWorker=True --overwrite + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UTcMHZWDrz node/gke-jen-pxc-2227-2da0e2d-default-pool-455a2d45-drbw not labeled + cat /tmp/tmp.kXysJd8eoX + rm /tmp/tmp.UTcMHZWDrz /tmp/tmp.kXysJd8eoX + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2227-2da0e2db#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_OUT=/tmp/tmp.XkfFOsBvM1 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29185~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + local LAST_ERR=/tmp/tmp.QcmToblrGu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XkfFOsBvM1 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.QcmToblrGu + rm /tmp/tmp.XkfFOsBvM1 /tmp/tmp.QcmToblrGu + return 0 + wait_all_backups ++ get_running_backups_amount ++ grep -vE ':Succeeded|:Failed' ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hyOlZDHQ9A +++ mktemp ++ local LAST_ERR=/tmp/tmp.YNMUFFFts6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hyOlZDHQ9A ++ cat /tmp/tmp.YNMUFFFts6 ++ rm /tmp/tmp.hyOlZDHQ9A /tmp/tmp.YNMUFFFts6 ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ grep :Failed ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pKx61OCJJM +++ mktemp ++ local LAST_ERR=/tmp/tmp.Tqi5iu3zuO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pKx61OCJJM ++ cat /tmp/tmp.Tqi5iu3zuO ++ rm /tmp/tmp.pKx61OCJJM /tmp/tmp.Tqi5iu3zuO ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-2 ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.hFYDleGeoE ++ mktemp + local LAST_ERR=/tmp/tmp.DNw2L1k0Tm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hFYDleGeoE NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-20251031174031-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251031174031-q6fav Running 39s + cat /tmp/tmp.DNw2L1k0Tm + rm /tmp/tmp.hFYDleGeoE /tmp/tmp.DNw2L1k0Tm + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.ARlnBjeNtR ++ mktemp + local LAST_ERR=/tmp/tmp.FCNl3rpTBS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ARlnBjeNtR NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-r4t8x 1/1 Running 0 8m25s pxc-client-857d976497-lvd7p 2/2 Running 0 7m13s scheduled-backup-proxysql-0 3/3 Running 0 7m10s scheduled-backup-proxysql-1 3/3 Running 0 6m49s scheduled-backup-pxc-0 1/1 Running 0 7m10s scheduled-backup-pxc-1 1/1 Running 0 5m54s scheduled-backup-pxc-2 1/1 Running 0 4m39s xb-cron-scheduled-backup-pvc-20251031174031-q6fav-78jdp 0/1 Completed 0 41s + cat /tmp/tmp.FCNl3rpTBS + rm /tmp/tmp.ARlnBjeNtR /tmp/tmp.FCNl3rpTBS + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.3LsvzKzyTZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.qR8gjhAkkh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3LsvzKzyTZ ++ cat /tmp/tmp.qR8gjhAkkh ++ rm /tmp/tmp.3LsvzKzyTZ /tmp/tmp.qR8gjhAkkh ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed +++ mktemp ++ wc -l ++ local LAST_OUT=/tmp/tmp.kwLnM6GxII +++ mktemp ++ local LAST_ERR=/tmp/tmp.mQlJMOmfBS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kwLnM6GxII ++ cat /tmp/tmp.mQlJMOmfBS ++ rm /tmp/tmp.kwLnM6GxII /tmp/tmp.mQlJMOmfBS ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2227-2da0e2db#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29185~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.4cU3iUcvwt ++ mktemp + local LAST_ERR=/tmp/tmp.9bozOwU6Ex + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4cU3iUcvwt perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.9bozOwU6Ex + rm /tmp/tmp.4cU3iUcvwt /tmp/tmp.9bozOwU6Ex + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ZqbFksb9JY + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29185~ + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_ERR=/tmp/tmp.MbpZc6N11b + local exit_status=0 + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2227-2da0e2db#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZqbFksb9JY perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.MbpZc6N11b + rm /tmp/tmp.ZqbFksb9JY /tmp/tmp.MbpZc6N11b + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.MMFgYHyQ5J +++ mktemp ++ local LAST_ERR=/tmp/tmp.RdFOSMslhk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MMFgYHyQ5J ++ cat /tmp/tmp.RdFOSMslhk ++ rm /tmp/tmp.MMFgYHyQ5J /tmp/tmp.RdFOSMslhk ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pq2RJnH9uz +++ mktemp ++ local LAST_ERR=/tmp/tmp.PmLD4xg2PI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Pq2RJnH9uz ++ cat /tmp/tmp.PmLD4xg2PI ++ rm /tmp/tmp.Pq2RJnH9uz /tmp/tmp.PmLD4xg2PI ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-0 ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scheduled-backup-pxc-1 ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.Txk1WZ9Bhn ++ mktemp + local LAST_ERR=/tmp/tmp.YW2Hd4x0e2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Txk1WZ9Bhn NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251031174238-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-10-31-17:42:38-full Running 32s cron-scheduled-backup-pvc-20251031174031-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251031174031-q6fav Succeeded 2m3s 2m39s + cat /tmp/tmp.YW2Hd4x0e2 + rm /tmp/tmp.Txk1WZ9Bhn /tmp/tmp.YW2Hd4x0e2 + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.dxdL4pLZWP ++ mktemp + local LAST_ERR=/tmp/tmp.Km9NdueMPp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dxdL4pLZWP NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-r4t8x 1/1 Running 0 10m pxc-client-857d976497-lvd7p 2/2 Running 0 9m12s scheduled-backup-proxysql-0 3/3 Running 0 9m9s scheduled-backup-proxysql-1 3/3 Running 0 8m48s scheduled-backup-pxc-0 1/1 Running 0 9m9s scheduled-backup-pxc-1 1/1 Running 0 7m53s scheduled-backup-pxc-2 1/1 Running 0 6m38s xb-cron-scheduled-backup-aws-s3-20251031174238-q6fav-k84n5 1/1 Running 0 33s xb-cron-scheduled-backup-pvc-20251031174031-q6fav-78jdp 0/1 Completed 0 2m40s + cat /tmp/tmp.Km9NdueMPp + rm /tmp/tmp.dxdL4pLZWP /tmp/tmp.Km9NdueMPp + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.S6QXoOZRY7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.TQ0ouf6spm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.S6QXoOZRY7 ++ cat /tmp/tmp.TQ0ouf6spm ++ rm /tmp/tmp.S6QXoOZRY7 /tmp/tmp.TQ0ouf6spm ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.THesljvc11 +++ mktemp ++ local LAST_ERR=/tmp/tmp.9UmcDtASUx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.THesljvc11 ++ cat /tmp/tmp.9UmcDtASUx ++ rm /tmp/tmp.THesljvc11 /tmp/tmp.9UmcDtASUx ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for minio storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for minio storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29185~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_OUT=/tmp/tmp.kaW27gezo3 + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2227-2da0e2db#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ mktemp + local LAST_ERR=/tmp/tmp.Bp9HSntapL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kaW27gezo3 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.Bp9HSntapL + rm /tmp/tmp.kaW27gezo3 /tmp/tmp.Bp9HSntapL + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_OUT=/tmp/tmp.ZyMB7tKcHg + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2227-2da0e2db#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29185~ + local LAST_ERR=/tmp/tmp.3PyGBhREAL + local exit_status=0 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZyMB7tKcHg perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.3PyGBhREAL + rm /tmp/tmp.ZyMB7tKcHg /tmp/tmp.3PyGBhREAL + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.nXcUXzpDDE +++ mktemp ++ local LAST_ERR=/tmp/tmp.oyaodCNjPw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nXcUXzpDDE ++ cat /tmp/tmp.oyaodCNjPw ++ rm /tmp/tmp.nXcUXzpDDE /tmp/tmp.oyaodCNjPw ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.UwAoeXTqd4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.G6UmTsqRW6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UwAoeXTqd4 ++ cat /tmp/tmp.G6UmTsqRW6 ++ rm /tmp/tmp.UwAoeXTqd4 /tmp/tmp.G6UmTsqRW6 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met waiting for pod/scheduled-backup-pxc-2 to become Ready.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.0ooSnX5P18 ++ mktemp + local LAST_ERR=/tmp/tmp.oVSo1PiirE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0ooSnX5P18 NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-20251031174238-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2025-10-31-17:42:38-full Succeeded 97s 2m19s cron-scheduled-backup-minio-20251031174435-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2025-10-31-17:44:35-full Running 22s cron-scheduled-backup-pvc-20251031174031-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-20251031174031-q6fav Succeeded 3m50s 4m26s + cat /tmp/tmp.oVSo1PiirE + rm /tmp/tmp.0ooSnX5P18 /tmp/tmp.oVSo1PiirE + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.o6HXpoTMVZ ++ mktemp + local LAST_ERR=/tmp/tmp.eUDBTFUMVi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.o6HXpoTMVZ NAME READY STATUS RESTARTS AGE minio-service-55fcc5d75f-r4t8x 1/1 Running 0 12m pxc-client-857d976497-lvd7p 2/2 Running 0 10m scheduled-backup-proxysql-0 3/3 Running 0 10m scheduled-backup-proxysql-1 3/3 Running 0 10m scheduled-backup-pxc-0 1/1 Running 0 10m scheduled-backup-pxc-1 1/1 Running 0 9m40s scheduled-backup-pxc-2 1/1 Running 0 8m25s xb-cron-scheduled-backup-aws-s3-20251031174238-q6fav-k84n5 0/1 Completed 0 2m20s xb-cron-scheduled-backup-minio-20251031174435-q6fav-bvp68 1/1 Running 0 23s xb-cron-scheduled-backup-pvc-20251031174031-q6fav-78jdp 0/1 Completed 0 4m27s + cat /tmp/tmp.eUDBTFUMVi + rm /tmp/tmp.o6HXpoTMVZ /tmp/tmp.eUDBTFUMVi + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.xF2YgvzEXp +++ mktemp ++ local LAST_ERR=/tmp/tmp.LhFUfCfgHE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xF2YgvzEXp ++ cat /tmp/tmp.LhFUfCfgHE ++ rm /tmp/tmp.xF2YgvzEXp /tmp/tmp.LhFUfCfgHE ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.TmgHJr4a16 +++ mktemp ++ local LAST_ERR=/tmp/tmp.iw54upFqWq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TmgHJr4a16 ++ cat /tmp/tmp.iw54upFqWq ++ rm /tmp/tmp.TmgHJr4a16 /tmp/tmp.iw54upFqWq ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for gcs storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for gcs storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.nJPNxWuEU8 + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2227-2da0e2db#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.2kem2TycCh + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29185~ + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nJPNxWuEU8 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.2kem2TycCh + rm /tmp/tmp.nJPNxWuEU8 /tmp/tmp.2kem2TycCh + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2227/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2227-2da0e2db#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-29185~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_OUT=/tmp/tmp.6daxQhkzae + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.hxxnOCxTL0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6daxQhkzae perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.hxxnOCxTL0 + rm /tmp/tmp.6daxQhkzae /tmp/tmp.hxxnOCxTL0 + return 0 + wait_all_backups ++ get_running_backups_amount ++ grep -vE ':Succeeded|:Failed' ++ wc -l ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9672SEr2n9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.wl3MGa7fh2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9672SEr2n9 ++ cat /tmp/tmp.wl3MGa7fh2 ++ rm /tmp/tmp.9672SEr2n9 /tmp/tmp.wl3MGa7fh2 ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.3v8Rui8qAJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.CehvAYPq4y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3v8Rui8qAJ ++ cat /tmp/tmp.CehvAYPq4y ++ rm /tmp/tmp.3v8Rui8qAJ /tmp/tmp.CehvAYPq4y ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met waiting for pod/scheduled-backup-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-1 waiting for pod/scheduled-backup-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-2 waiting for pod/scheduled-backup-pxc-2 to become Ready.Name: scheduled-backup-pxc-2 Namespace: scheduled-backup-29185 Priority: 0 Service Account: default Node: Labels: app.kubernetes.io/component=pxc app.kubernetes.io/instance=scheduled-backup app.kubernetes.io/managed-by=percona-xtradb-cluster-operator app.kubernetes.io/name=percona-xtradb-cluster app.kubernetes.io/part-of=percona-xtradb-cluster apps.kubernetes.io/pod-index=2 controller-revision-hash=scheduled-backup-pxc-64966478ff statefulset.kubernetes.io/pod-name=scheduled-backup-pxc-2 Annotations: cloud.google.com/cluster_autoscaler_unhelpable_since: 2025-10-31T17:46:40+0000 cloud.google.com/cluster_autoscaler_unhelpable_until: Inf kubectl.kubernetes.io/default-container: pxc percona.com/configuration-hash: d41d8cd98f00b204e9800998ecf8427e percona.com/ssl-hash: cfd6a52398268173b51d7cdb1331c09a percona.com/ssl-internal-hash: 751e59a1c1d94ac67f9d6bb6e756acdc Status: Pending IP: IPs: Controlled By: StatefulSet/scheduled-backup-pxc Init Containers: pxc-init: Image: perconalab/percona-xtradb-cluster-operator:PR-2227-2da0e2db Port: Host Port: Command: /pxc-init-entrypoint.sh Limits: cpu: 50m memory: 50M Requests: cpu: 50m memory: 50M Environment: Mounts: /var/lib/mysql from datadir (rw) /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-dtxnm (ro) Containers: pxc: Image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7 Ports: 3306/TCP (mysql), 4444/TCP (sst), 4567/TCP (write-set), 4568/TCP (ist), 33062/TCP (mysql-admin), 33060/TCP (mysqlx) Host Ports: 0/TCP (mysql), 0/TCP (sst), 0/TCP (write-set), 0/TCP (ist), 0/TCP (mysql-admin), 0/TCP (mysqlx) Command: /var/lib/mysql/pxc-entrypoint.sh Args: mysqld Liveness: exec [/var/lib/mysql/liveness-check.sh] delay=300s timeout=5s period=10s #success=1 #failure=3 Readiness: exec [/var/lib/mysql/readiness-check.sh] delay=15s timeout=15s period=30s #success=1 #failure=5 Environment Variables from: scheduled-backup-env-vars-pxc Secret Optional: true Environment: PXC_SERVICE: scheduled-backup-pxc-unready MONITOR_HOST: % MYSQL_ROOT_PASSWORD: Optional: false XTRABACKUP_PASSWORD: Optional: false MONITOR_PASSWORD: Optional: false CLUSTER_HASH: 4473288 OPERATOR_ADMIN_PASSWORD: Optional: false LIVENESS_CHECK_TIMEOUT: 5 READINESS_CHECK_TIMEOUT: 15 DEFAULT_AUTHENTICATION_PLUGIN: mysql_native_password MYSQL_NOTIFY_SOCKET: /var/lib/mysql/notify.sock MYSQL_STATE_FILE: /var/lib/mysql/mysql.state Mounts: /etc/my.cnf.d from auto-config (rw) /etc/mysql/init-file from mysql-init-file (rw) /etc/mysql/mysql-users-secret from mysql-users-secret-file (rw) /etc/mysql/ssl from ssl (rw) /etc/mysql/ssl-internal from ssl-internal (rw) /etc/mysql/vault-keyring-secret from vault-keyring-secret (rw) /etc/percona-xtradb-cluster.conf.d from config (rw) /tmp from tmp (rw) /var/lib/mysql from datadir (rw) /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-dtxnm (ro) Conditions: Type Status PodScheduled False Volumes: datadir: Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace) ClaimName: datadir-scheduled-backup-pxc-2 ReadOnly: false tmp: Type: EmptyDir (a temporary directory that shares a pod's lifetime) Medium: SizeLimit: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: scheduled-backup-pxc Optional: true ssl-internal: Type: Secret (a volume populated by a Secret) SecretName: some-name-ssl-internal Optional: true ssl: Type: Secret (a volume populated by a Secret) SecretName: some-name-ssl Optional: false auto-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: auto-scheduled-backup-pxc Optional: true vault-keyring-secret: Type: Secret (a volume populated by a Secret) SecretName: scheduled-backup-vault Optional: true mysql-users-secret-file: Type: Secret (a volume populated by a Secret) SecretName: internal-scheduled-backup Optional: false mysql-init-file: Type: Secret (a volume populated by a Secret) SecretName: scheduled-backup-mysql-init Optional: true kube-api-access-dtxnm: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt Optional: false DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedScheduling 8s default-scheduler 0/3 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }, 2 node(s) didn't match pod anti-affinity rules. preemption: 0/3 nodes are available: 1 Preemption is not helpful for scheduling, 2 No preemption victims found for incoming pod. Warning FailedScheduling 3s (x2 over 6s) default-scheduler 0/3 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }, 2 node(s) didn't match pod anti-affinity rules. preemption: 0/3 nodes are available: 1 Preemption is not helpful for scheduling, 2 No preemption victims found for incoming pod. Normal NotTriggerScaleUp 8s cluster-autoscaler pod didn't trigger scale-up: 2025-10-31T17:32:06.958Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.31.13-gke.1123000"} 2025-10-31T17:32:06.959Z INFO setup Manager starting up {"gitCommit": "2da0e2db322d00c546daefbee2d2fcdc1a6fec5c", "gitBranch": "PR-2227-2da0e2db", "buildTime": "2025-10-31T15:51:42Z", "goVersion": "go1.25.3", "os": "linux", "arch": "amd64"} 2025-10-31T17:32:06.962Z INFO setup Registering Components. 2025-10-31T17:32:07.230Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2025-10-31T17:32:07.231Z INFO setup Starting the Cmd. 2025-10-31T17:32:07.231Z INFO controller-runtime.metrics Starting metrics server 2025-10-31T17:32:07.231Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2025-10-31T17:32:07.231Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-10-31T17:32:07.232Z INFO controller-runtime.webhook Starting webhook server 2025-10-31T17:32:07.232Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2025-10-31T17:32:07.232Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-10-31T17:32:07.232Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2025-10-31T17:32:07.433Z INFO attempting to acquire leader lease pxc-operator/08db1feb.percona.com... 2025-10-31T17:32:07.621Z INFO successfully acquired lease pxc-operator/08db1feb.percona.com 2025-10-31T17:32:07.621Z DEBUG events percona-xtradb-cluster-operator-554c4f86-ncnv4_8b620820-e403-4502-8cc6-635c23cfa314 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"fb3d711f-89c5-482a-a4ea-5519922f052f","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1761931927589775009"}, "reason": "LeaderElection"} 2025-10-31T17:32:07.621Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.Secret"} 2025-10-31T17:32:07.621Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"} 2025-10-31T17:32:07.621Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2025-10-31T17:32:07.622Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2025-10-31T17:32:07.722Z INFO Starting Controller {"controller": "pxc-controller"} 2025-10-31T17:32:07.722Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2025-10-31T17:32:07.722Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2025-10-31T17:32:07.722Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2025-10-31T17:32:07.722Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2025-10-31T17:32:07.722Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2025-10-31T17:34:02.096Z INFO Set CR version {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "c7fe30c6-4568-433b-abf1-d35ee8ab4e90", "version": "1.19.0"} 2025-10-31T17:34:02.520Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "c7fe30c6-4568-433b-abf1-d35ee8ab4e90", "object": "scheduled-backup-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2025-10-31T17:34:02.558Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "c7fe30c6-4568-433b-abf1-d35ee8ab4e90", "object": "scheduled-backup-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2025-10-31T17:34:02.619Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "c7fe30c6-4568-433b-abf1-d35ee8ab4e90", "object": "scheduled-backup-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-31T17:34:02.667Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "c7fe30c6-4568-433b-abf1-d35ee8ab4e90", "object": "scheduled-backup-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-31T17:34:02.723Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "c7fe30c6-4568-433b-abf1-d35ee8ab4e90", "object": "scheduled-backup-proxysql", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-31T17:34:02.822Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "c7fe30c6-4568-433b-abf1-d35ee8ab4e90", "object": "scheduled-backup-proxysql-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2025-10-31T17:34:03.295Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "06f16591-cba7-4b05-848d-da3fbdee3606", "object": "scheduled-backup-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2025-10-31T17:34:03.317Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "06f16591-cba7-4b05-848d-da3fbdee3606", "object": "scheduled-backup-proxysql", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2025-10-31T17:35:19.462Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "211ba46b-8124-46c0-b596-8c5832a1d4d1", "user": "operator"} 2025-10-31T17:35:19.497Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "211ba46b-8124-46c0-b596-8c5832a1d4d1", "user": "monitor"} 2025-10-31T17:35:19.534Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "211ba46b-8124-46c0-b596-8c5832a1d4d1"} 2025-10-31T17:35:19.572Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "211ba46b-8124-46c0-b596-8c5832a1d4d1", "user": "xtrabackup"} 2025-10-31T17:35:19.601Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "211ba46b-8124-46c0-b596-8c5832a1d4d1"} 2025-10-31T17:35:19.609Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "211ba46b-8124-46c0-b596-8c5832a1d4d1", "err": "get primary pxc pod: not found"} 2025-10-31T17:35:24.436Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "8563e86d-5cfa-44df-8cc4-b87954982997", "err": "get primary pxc pod: not found"} 2025-10-31T17:35:29.573Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "78886731-2619-4b37-bd33-1742eedb38b5", "err": "get primary pxc pod: not found"} 2025-10-31T17:35:34.704Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "6dafbdcf-1b44-4f8a-81c2-d1205bc7b4a9", "err": "get primary pxc pod: not found"} 2025-10-31T17:37:51.118Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "0ac0865f-7607-4f05-b1f8-3ba7c07d728a", "user": "root"} 2025-10-31T17:37:51.163Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "0ac0865f-7607-4f05-b1f8-3ba7c07d728a", "user": "replication"} 2025-10-31T17:37:51.213Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "0ac0865f-7607-4f05-b1f8-3ba7c07d728a", "new version": "5.7.44-48-57"} 2025-10-31T17:37:53.524Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "0ac0865f-7607-4f05-b1f8-3ba7c07d728a"} 2025-10-31T17:37:57.479Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "27733d36-445a-4422-8656-adf3d029299c"} 2025-10-31T17:38:02.721Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "473e1b7e-09d2-4064-b68e-ed180403a614"} 2025-10-31T17:38:08.381Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "061268d8-5292-4336-b3a4-9854ee97f511"} 2025-10-31T17:38:13.304Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "659bf944-c6b8-4286-8fa1-fde0f0b337c9"} 2025-10-31T17:38:18.326Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "f210f8d4-d1ba-4f1b-a447-7a08ea3d2c10"} 2025-10-31T17:38:24.397Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "ea6f1ca7-b7cb-4745-87f3-daab75f4b247"} 2025-10-31T17:38:29.678Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "274b7ead-0bba-4a01-bf95-2ce069b5a2b5"} 2025-10-31T17:38:34.868Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "385a59cf-cdae-4827-b625-847280bb77ae"} 2025-10-31T17:38:40.137Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "be30a9d3-2165-45d1-8d81-0ae0a21e3d29"} 2025-10-31T17:38:45.429Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "21e2b095-dafa-43ca-9758-75e8efba29e5"} 2025-10-31T17:38:50.445Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "7990b91d-8ad0-464f-90d9-eae5813840ff"} 2025-10-31T17:38:55.926Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "123c0d18-73ce-479f-9ae7-8656511d0cad"} 2025-10-31T17:39:00.883Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "781733d0-61af-4826-bc3e-08f728c99491"} 2025-10-31T17:39:06.340Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "b7ed8430-894d-402f-ac3f-bb1257ed41b2"} 2025-10-31T17:39:11.635Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "1793954e-ddc7-4df8-9119-ff19df85419b"} 2025-10-31T17:39:16.885Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "2d3eb954-fe4a-4d2b-967d-2bcc5ce3e1c5"} 2025-10-31T17:39:21.888Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "ae8cf4c7-ff1e-4d38-a71c-cc3aa4263cae"} 2025-10-31T17:39:27.120Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "fddcdcc2-f0ff-45e6-b152-21d794f2f6b4"} 2025-10-31T17:39:31.065Z INFO Creating or updating backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "a9e291e0-bb89-45cd-ae38-ff4026e612ba", "name": "508bb-each-min-pvc", "schedule": "* * * * *"} 2025-10-31T17:39:32.441Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "a9e291e0-bb89-45cd-ae38-ff4026e612ba"} 2025-10-31T17:39:33.740Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "f693dcd4-a2a2-44f5-b97e-2b13f5535d8e"} 2025-10-31T17:39:37.551Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "eaaaabe5-7ed4-407c-9570-4f7e7b817fbc"} 2025-10-31T17:39:42.935Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "7602a98a-7e2a-4fc1-ae0b-525fe7b554d0"} 2025-10-31T17:39:47.909Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "bd3497b2-3883-4b27-a09d-9d374f0eb43d"} 2025-10-31T17:39:54.104Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "6a9236fb-c91c-411b-978d-7c61598f0319"} 2025-10-31T17:39:59.582Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "89af5671-00d4-4f76-9a6e-cdc469e87fbb"} 2025-10-31T17:40:04.848Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "a61e1177-eb88-4d70-bb88-aad3c412cce3"} 2025-10-31T17:40:09.721Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "63f7f556-af7c-4086-969b-f8524a0b2d0c"} 2025-10-31T17:40:15.389Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "efde1399-b9c9-4a12-b2a4-7d1de3951202"} 2025-10-31T17:40:20.777Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "7483bcfc-d1f9-46f0-95b2-010462277eb5"} 2025-10-31T17:40:25.960Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "caf47360-cae6-4b20-a1c7-dabd9bbd56b7"} 2025-10-31T17:40:31.136Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "6cc226e0-9748-4bc1-8b5f-dd4c3d72e329", "cluster": "scheduled-backup", "storage": "pvc", "allowed": true} 2025-10-31T17:40:31.137Z INFO Creating a new volume for backup {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "6cc226e0-9748-4bc1-8b5f-dd4c3d72e329", "Namespace": "scheduled-backup-29185", "Name": "xb-cron-scheduled-backup-pvc-20251031174031-q6fav"} 2025-10-31T17:40:31.208Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "3687e7fd-7ff6-41c0-9eb7-38ac15f8396c"} 2025-10-31T17:40:31.292Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "6cc226e0-9748-4bc1-8b5f-dd4c3d72e329", "namespace": "scheduled-backup-29185", "name": "xb-cron-scheduled-backup-pvc-20251031174031-q6fav"} 2025-10-31T17:40:36.292Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "52ecf3ad-e4ab-4321-bc51-ba5cac8d0ba8", "cluster": "scheduled-backup", "storage": "pvc", "allowed": true} 2025-10-31T17:40:36.362Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "eac7f670-4b02-4c20-9814-88132016143c", "cluster": "scheduled-backup", "storage": "pvc", "allowed": true} 2025-10-31T17:40:36.503Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "6b7f033c-b094-4ffc-9afe-08b8d5a7cbe6"} 2025-10-31T17:40:37.943Z INFO deleting outdated backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "b25a2b87-7c4a-44ee-b9ca-506db8bf9c6e", "name": "508bb-each-min-pvc"} 2025-10-31T17:40:39.386Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "b25a2b87-7c4a-44ee-b9ca-506db8bf9c6e"} 2025-10-31T17:40:41.352Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "086cec50-a048-4397-9f0a-c703c8085603", "cluster": "scheduled-backup", "storage": "pvc", "allowed": true} 2025-10-31T17:40:41.750Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "cd46c21b-0b1a-4e5b-bb3a-9330c2ba4e1b"} 2025-10-31T17:40:46.391Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "dd4b0259-f41a-47e6-917a-f58b28f469da", "cluster": "scheduled-backup", "storage": "pvc", "allowed": true} 2025-10-31T17:40:47.042Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "c7854089-a45c-42f4-8c25-edf7ed1fdac5"} 2025-10-31T17:40:51.438Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "51011ce2-0ea9-4637-8734-1af99b0da40f", "cluster": "scheduled-backup", "storage": "pvc", "allowed": true} 2025-10-31T17:40:52.335Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "829e0a12-d324-4f6e-89b2-2674256bd04f"} 2025-10-31T17:40:56.478Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "359fa0ea-b1d8-4421-8e77-85573dfe981b", "cluster": "scheduled-backup", "storage": "pvc", "allowed": true} 2025-10-31T17:40:57.518Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "394aef6c-bfdc-4e83-a70d-099d3e3368a8"} 2025-10-31T17:41:01.518Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "150a61c1-5957-4b8e-927c-028cf3773128", "cluster": "scheduled-backup", "storage": "pvc", "allowed": true} 2025-10-31T17:41:02.529Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "0b298726-3caf-4723-a35c-c916452f2521"} 2025-10-31T17:41:06.565Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "64a4961c-0fbe-47a3-83c4-7d5f356c60c7", "cluster": "scheduled-backup", "storage": "pvc", "allowed": true} 2025-10-31T17:41:08.302Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "07119ea2-7ae6-45d2-b5df-379112606e1d"} 2025-10-31T17:41:11.609Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "b8505abd-3919-4c20-9e76-2ffb3ee46fe4", "cluster": "scheduled-backup", "storage": "pvc", "allowed": true} 2025-10-31T17:41:11.655Z INFO Backup succeeded {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "b8505abd-3919-4c20-9e76-2ffb3ee46fe4", "job": "xb-cron-scheduled-backup-pvc-20251031174031-q6fav"} 2025-10-31T17:41:11.655Z DEBUG Removing mysql-init secret {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-pvc-20251031174031-q6fav", "reconcileID": "b8505abd-3919-4c20-9e76-2ffb3ee46fe4", "job": "xb-cron-scheduled-backup-pvc-20251031174031-q6fav", "secret": "scheduled-backup-mysql-init"} 2025-10-31T17:41:13.236Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "2eaba3bb-66df-4ed5-9f43-4604748015b6"} 2025-10-31T17:41:18.686Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "96f84832-171e-40a9-9388-0a21fa292252"} 2025-10-31T17:41:24.454Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "32c6f7e2-c26c-4a0a-9831-58cfd7d9b4fc"} 2025-10-31T17:41:29.802Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "15ac0c36-d7d7-4332-8fa2-8d727ba40aa1"} 2025-10-31T17:41:34.825Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "2fce338a-021b-4d63-b2a9-449836bc353a"} 2025-10-31T17:41:38.515Z INFO Creating or updating backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "c8f70035-c604-45f5-bace-d35000057c1e", "name": "508bb-each-min-aws-s3", "schedule": "* * * * *"} 2025-10-31T17:41:39.804Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "c8f70035-c604-45f5-bace-d35000057c1e"} 2025-10-31T17:41:41.413Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "0a3b78d7-27da-4fd2-8664-b2fab1f7eac5"} 2025-10-31T17:41:45.580Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "57fec327-0f7c-4959-8665-7f189c8b10a1"} 2025-10-31T17:41:51.070Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "280529e8-d1c9-4e3d-8e62-5e46330cdb36"} 2025-10-31T17:41:56.135Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "b31aa502-00a7-4b76-9408-6e96dba956c0"} 2025-10-31T17:42:01.708Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "29c4c4a4-6f7d-4521-a985-039395c08180"} 2025-10-31T17:42:06.870Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "5ef8f1fb-4400-42aa-a558-6cca8127e0d6"} 2025-10-31T17:42:12.097Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "e776b4c1-94a6-44de-96a5-fe557eb43a4d"} 2025-10-31T17:42:17.463Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "c8cda292-5e32-4d2a-9cac-a518343fca26"} 2025-10-31T17:42:23.207Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "ed43f7bf-d398-4358-8d4f-f2dfb1fa85c5"} 2025-10-31T17:42:27.854Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "ae8d8f6a-5dc6-4383-8d03-0d0e8ce3e2b6"} 2025-10-31T17:42:33.163Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "e3a09d00-3cc3-4ac1-89d2-db4303aaa1ef"} 2025-10-31T17:42:38.082Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "b321e97a-1d48-4004-a436-a35e36364df6", "cluster": "scheduled-backup", "storage": "aws-s3", "allowed": true} 2025-10-31T17:42:38.283Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "b321e97a-1d48-4004-a436-a35e36364df6", "namespace": "scheduled-backup-29185", "name": "xb-cron-scheduled-backup-aws-s3-20251031174238-q6fav"} 2025-10-31T17:42:38.693Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "ee6f5ab9-2052-444b-a1a0-af6f562c6e86"} 2025-10-31T17:42:43.165Z INFO deleting outdated backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "5a7d5a0b-9977-4a66-ab58-be69bf37d3d4", "name": "508bb-each-min-aws-s3"} 2025-10-31T17:42:43.284Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "062e1a82-0648-4005-9a91-20b4af743c32", "cluster": "scheduled-backup", "storage": "aws-s3", "allowed": true} 2025-10-31T17:42:43.371Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "586c67eb-79db-475c-b8fc-eb32efba94f0", "cluster": "scheduled-backup", "storage": "aws-s3", "allowed": true} 2025-10-31T17:42:43.524Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "3006bb8b-eab6-4b4f-80ea-8efd78af933e"} 2025-10-31T17:42:45.201Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "20397b28-9994-4e6c-a1db-b9abe7272e4d"} 2025-10-31T17:42:48.361Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "0b72399d-0d7e-46f1-938f-bff87d65c4d5", "cluster": "scheduled-backup", "storage": "aws-s3", "allowed": true} 2025-10-31T17:42:48.985Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "073e019a-7984-4d95-b86d-fb29b0978e28"} 2025-10-31T17:42:53.399Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "cb0214c5-9b20-4e11-8ed3-5dc12c315eb9", "cluster": "scheduled-backup", "storage": "aws-s3", "allowed": true} 2025-10-31T17:42:53.799Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "0ba529be-d0b3-49ff-9e3e-2d11deee1d67"} 2025-10-31T17:42:58.431Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "694626e4-b7b0-42bf-bcd7-c62a8307229b", "cluster": "scheduled-backup", "storage": "aws-s3", "allowed": true} 2025-10-31T17:42:59.474Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "6cd4c1b1-bfb1-4179-834b-e0169a4e242a"} 2025-10-31T17:43:03.475Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "9a6ecbf5-64db-4928-b4d7-8686adf4db67", "cluster": "scheduled-backup", "storage": "aws-s3", "allowed": true} 2025-10-31T17:43:04.690Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "cb00e155-76f0-4703-b01c-8eed3e7a915c"} 2025-10-31T17:43:08.511Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "c85f6ff0-7ffb-4373-b0a0-592259f41749", "cluster": "scheduled-backup", "storage": "aws-s3", "allowed": true} 2025-10-31T17:43:09.939Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "38f1ed22-cdbf-476e-b71c-52edf5875d8d"} 2025-10-31T17:43:13.557Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "49219cc1-b0b4-4ad6-b053-d6b505e9801e", "cluster": "scheduled-backup", "storage": "aws-s3", "allowed": true} 2025-10-31T17:43:15.199Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "ef8ca9f2-c88c-49e5-b9a7-309c9723e022"} 2025-10-31T17:43:18.595Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "777193e8-f0c1-463d-9222-7ee8684aee78", "cluster": "scheduled-backup", "storage": "aws-s3", "allowed": true} 2025-10-31T17:43:20.526Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "8c1ef67a-a9c7-4e8f-95b3-bf046ac628e5"} 2025-10-31T17:43:23.632Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "eff2a105-c508-4b53-9c3f-36fafbdcf0e4", "cluster": "scheduled-backup", "storage": "aws-s3", "allowed": true} 2025-10-31T17:43:23.674Z INFO Backup succeeded {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "eff2a105-c508-4b53-9c3f-36fafbdcf0e4", "job": "xb-cron-scheduled-backup-aws-s3-20251031174238-q6fav"} 2025-10-31T17:43:23.674Z DEBUG Removing mysql-init secret {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-aws-s3-20251031174238-q6fav", "reconcileID": "eff2a105-c508-4b53-9c3f-36fafbdcf0e4", "job": "xb-cron-scheduled-backup-aws-s3-20251031174238-q6fav", "secret": "scheduled-backup-mysql-init"} 2025-10-31T17:43:25.712Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "128d86e1-8247-43d6-bb16-b840f978853f"} 2025-10-31T17:43:30.754Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "8ccd22af-8afe-4c62-9e76-9304dadee799"} 2025-10-31T17:43:35.532Z INFO Creating or updating backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "8460c74d-816b-4ef8-ab72-a55c605f1f7e", "name": "508bb-each-min-minio", "schedule": "* * * * *"} 2025-10-31T17:43:36.265Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "b373a14d-1804-4c44-8c21-6ac6be0c0bbc"} 2025-10-31T17:43:37.847Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "53b59be6-ec9a-41f7-b1df-2b6e9aafebb0"} 2025-10-31T17:43:41.478Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "9d0d1455-a1dd-4f8b-aa88-ebdaa743b9bb"} 2025-10-31T17:43:46.748Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "b1fd5206-6d47-409d-90a0-df62d80bb266"} 2025-10-31T17:43:51.979Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "d9f70ac9-67ba-45a1-aca8-857cbb1d6eda"} 2025-10-31T17:43:57.034Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "565eb191-93b6-4169-bee8-2b474534f57a"} 2025-10-31T17:44:02.506Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "e2db03e1-5660-4bf9-9ea6-7e6f7e52de98"} 2025-10-31T17:44:08.176Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "718631a3-55f5-4ab2-ae10-e7e4a39f4178"} 2025-10-31T17:44:12.980Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "67723538-37ca-4716-bec2-7e348bb0993d"} 2025-10-31T17:44:17.999Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "80f348c8-cc00-46b4-b5d5-40cc9bc73783"} 2025-10-31T17:44:24.141Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "71b16c6b-64d9-4fe1-a020-b03eeb085a85"} 2025-10-31T17:44:29.259Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "cbb44fe0-01e6-47ef-8eb3-791bd09795de"} 2025-10-31T17:44:34.659Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "63e76f59-4c90-4d40-9a78-082fe48bb4ba"} 2025-10-31T17:44:35.033Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-minio-20251031174435-q6fav", "reconcileID": "4ad026d1-95dc-461d-a178-47ee45daa0cf", "cluster": "scheduled-backup", "storage": "minio", "allowed": true} 2025-10-31T17:44:35.064Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-minio-20251031174435-q6fav", "reconcileID": "4ad026d1-95dc-461d-a178-47ee45daa0cf", "namespace": "scheduled-backup-29185", "name": "xb-cron-scheduled-backup-minio-20251031174435-q6fav"} 2025-10-31T17:44:39.203Z INFO deleting outdated backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "2da10047-c38a-4a7c-8c39-49b6bab7778c", "name": "508bb-each-min-minio"} 2025-10-31T17:44:39.870Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "c5493724-2bca-44a1-93c7-7df9c2821767"} 2025-10-31T17:44:40.065Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-minio-20251031174435-q6fav", "reconcileID": "b1f879c5-d937-4973-b9a5-69ccfcbf3772", "cluster": "scheduled-backup", "storage": "minio", "allowed": true} 2025-10-31T17:44:40.200Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-minio-20251031174435-q6fav", "reconcileID": "58863f01-8496-4bff-9a71-64294ad0ed3c", "cluster": "scheduled-backup", "storage": "minio", "allowed": true} 2025-10-31T17:44:41.504Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "0ca5978e-a01f-4555-933e-d9ddffbe0ef7"} 2025-10-31T17:44:45.016Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "d1c0e419-6916-4866-afea-036db29ba0a7"} 2025-10-31T17:44:45.192Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-minio-20251031174435-q6fav", "reconcileID": "24287d09-c994-45e9-b3cd-f5132e5c2e82", "cluster": "scheduled-backup", "storage": "minio", "allowed": true} 2025-10-31T17:44:50.232Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-minio-20251031174435-q6fav", "reconcileID": "25581f48-37f3-4f72-8bbd-80dbdccb7706", "cluster": "scheduled-backup", "storage": "minio", "allowed": true} 2025-10-31T17:44:50.411Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "362ea7d0-3785-4ec1-ac9c-02c5fa35c6c7"} 2025-10-31T17:44:55.267Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-minio-20251031174435-q6fav", "reconcileID": "adafb4cd-e6b9-4b1a-bc38-fdf9882502a8", "cluster": "scheduled-backup", "storage": "minio", "allowed": true} 2025-10-31T17:44:55.446Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "0c499bae-8797-4481-9cd3-732cc3b8f8bb"} 2025-10-31T17:45:00.312Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-minio-20251031174435-q6fav", "reconcileID": "43cdb30b-8267-4a93-b955-cb0233198dd3", "cluster": "scheduled-backup", "storage": "minio", "allowed": true} 2025-10-31T17:45:00.978Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "1373338e-efda-4b5a-a2ef-028acb13914b"} 2025-10-31T17:45:05.389Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-minio-20251031174435-q6fav", "reconcileID": "a9ec2c6d-f1ca-4afd-892a-d22827c4a3f7", "cluster": "scheduled-backup", "storage": "minio", "allowed": true} 2025-10-31T17:45:05.439Z INFO Backup succeeded {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-minio-20251031174435-q6fav", "reconcileID": "a9ec2c6d-f1ca-4afd-892a-d22827c4a3f7", "job": "xb-cron-scheduled-backup-minio-20251031174435-q6fav"} 2025-10-31T17:45:05.439Z DEBUG Removing mysql-init secret {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-minio-20251031174435-q6fav", "reconcileID": "a9ec2c6d-f1ca-4afd-892a-d22827c4a3f7", "job": "xb-cron-scheduled-backup-minio-20251031174435-q6fav", "secret": "scheduled-backup-mysql-init"} 2025-10-31T17:45:06.236Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "5c4fd3c4-bb4a-4db5-a06c-186d265f9157"} 2025-10-31T17:45:11.500Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "a60b186a-de33-4f48-8c62-feb065e684c4"} 2025-10-31T17:45:16.727Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "3f44df53-49e7-4448-b4ba-265e07c253ba"} 2025-10-31T17:45:22.086Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "6a6d47ee-6439-4f32-9116-2f7578a75fde"} 2025-10-31T17:45:22.510Z INFO Creating or updating backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "e4589c48-fd1f-46ce-b0ab-f5c3d6cb4bc8", "name": "508bb-each-min-gcp-cs", "schedule": "* * * * *"} 2025-10-31T17:45:23.918Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "e4589c48-fd1f-46ce-b0ab-f5c3d6cb4bc8"} 2025-10-31T17:45:27.060Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "bb0d0f2b-afe0-4b8a-a0c9-5709189752d3"} 2025-10-31T17:45:32.443Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "0c667ac0-7c03-460e-9326-be796692f843"} 2025-10-31T17:45:37.847Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "7971103f-15dc-4e0b-8b73-36a7c5b901f6"} 2025-10-31T17:45:42.974Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "aa0a9732-036b-4ddd-9140-4c0f66568d10"} 2025-10-31T17:45:48.214Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "4c50f272-c729-454a-959f-647f28eb53eb"} 2025-10-31T17:45:53.934Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "f163b472-42be-49dd-9de0-c4fd0edb1c06"} 2025-10-31T17:45:59.453Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "1ecda492-1d5b-4fe2-ab87-2786cf44c31a"} 2025-10-31T17:46:04.450Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "74f3b407-3bda-4cc1-bdad-7cd9a50718e7"} 2025-10-31T17:46:09.857Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "5573a6e6-6408-47fd-9c39-d65c97991fc0"} 2025-10-31T17:46:15.148Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "87b8dafe-7da9-450b-ae4c-b695e3a6e28f"} 2025-10-31T17:46:20.322Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "45dae986-8a57-42f0-b515-9edba1e8e953"} 2025-10-31T17:46:22.048Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "reconcileID": "c4853389-2e63-4b9b-bf68-26d4a42926da", "cluster": "scheduled-backup", "storage": "gcp-cs", "allowed": true} 2025-10-31T17:46:22.529Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "reconcileID": "c4853389-2e63-4b9b-bf68-26d4a42926da", "namespace": "scheduled-backup-29185", "name": "xb-cron-scheduled-backup-gcp-cs-20251031174622-q6fav"} 2025-10-31T17:46:22.718Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "reconcileID": "fe815527-54c3-4db4-9c19-71c4d7fec5c0", "cluster": "scheduled-backup", "storage": "gcp-cs", "allowed": true} 2025-10-31T17:46:25.624Z ERROR sync users {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "44c583e1-c778-4de6-be1a-a68f41fa2a6a", "error": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") / / ", "errorVerbose": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") / / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:966\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:867\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:869 2025-10-31T17:46:27.718Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "reconcileID": "4650234b-8277-48b9-b7e0-dd45ba4f21aa", "cluster": "scheduled-backup", "storage": "gcp-cs", "allowed": true} 2025-10-31T17:46:27.793Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "reconcileID": "d828d36f-1d1a-4378-bfd5-6d2f4ab39b43", "cluster": "scheduled-backup", "storage": "gcp-cs", "allowed": true} 2025-10-31T17:46:32.635Z INFO deleting outdated backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-29185", "name": "scheduled-backup", "reconcileID": "60ddf8c5-5ae8-4e29-b0f6-846a801da9d8", "name": "508bb-each-min-gcp-cs"} 2025-10-31T17:46:32.791Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "reconcileID": "5995f429-23be-46bf-8938-cef0b108de8a", "cluster": "scheduled-backup", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2025-10-31T17:46:37.792Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "reconcileID": "f195a40f-dd7e-4b96-b07d-e25b208a0cea", "cluster": "scheduled-backup", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2025-10-31T17:46:42.792Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "reconcileID": "c3e93995-2443-4f4c-9134-ed9fa6dcabc9", "cluster": "scheduled-backup", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2025-10-31T17:46:47.793Z INFO Suspending backup job {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "reconcileID": "1af15789-a4a5-4774-a726-8f3342f147ef", "job": "xb-cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "clusterStatus": "initializing", "readyPXC": 2} 2025-10-31T17:46:47.845Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "reconcileID": "1af15789-a4a5-4774-a726-8f3342f147ef", "cluster": "scheduled-backup", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2025-10-31T17:46:47.856Z INFO Suspending backup job {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "reconcileID": "5805196e-d9e4-49eb-8fde-87c54810b612", "job": "xb-cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "clusterStatus": "initializing", "readyPXC": 2} 2025-10-31T17:46:47.910Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-29185", "name": "cron-scheduled-backup-gcp-cs-20251031174622-q6fav", "reconcileID": "5805196e-d9e4-49eb-8fde-87c54810b612", "cluster": "scheduled-backup", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} max retry count 1 reached. something went wrong with operator or kubernetes cluster