Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/logs/scheduled-backup-8-0.log WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 + main + create_infra scheduled-backup-19207 + local ns=scheduled-backup-19207 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n scheduled-backup-18460 scheduled-backup --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/scheduled-backup patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.XH3IME321C ++ mktemp + local LAST_ERR=/tmp/tmp.GBCrga8bky + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XH3IME321C perconaxtradbcluster.pxc.percona.com "scheduled-backup" deleted + cat /tmp/tmp.GBCrga8bky + rm /tmp/tmp.XH3IME321C /tmp/tmp.GBCrga8bky + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.WQ0EQGJPON ++ mktemp + local LAST_ERR=/tmp/tmp.ycgJ5VMEGl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WQ0EQGJPON perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-aws-s3-2024514173314-q6fav" deleted perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-azure-blob-2024514173119-q6fav" deleted perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-gcp-cs-2024514172923-q6fav" deleted perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-minio-2024514172336-q6fav" deleted perconaxtradbclusterbackup.pxc.percona.com "cron-scheduled-backup-pvc-2024514171938-q6fav" deleted + cat /tmp/tmp.ycgJ5VMEGl + rm /tmp/tmp.WQ0EQGJPON /tmp/tmp.ycgJ5VMEGl + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.uvPtQ6eISg ++ mktemp + local LAST_ERR=/tmp/tmp.ZwfxKYirDH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uvPtQ6eISg perconaxtradbclusterrestore.pxc.percona.com "aws-s3-2024514173314-q6fav" deleted perconaxtradbclusterrestore.pxc.percona.com "azure-blob-2024514173119-q6fav" deleted perconaxtradbclusterrestore.pxc.percona.com "gcp-cs-2024514172923-q6fav" deleted perconaxtradbclusterrestore.pxc.percona.com "minio-2024514172336-q6fav" deleted perconaxtradbclusterrestore.pxc.percona.com "pvc-2024514171938-q6fav" deleted + cat /tmp/tmp.ZwfxKYirDH + rm /tmp/tmp.uvPtQ6eISg /tmp/tmp.ZwfxKYirDH + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.8dWwMg7tOQ ++ mktemp + local LAST_OUT=/tmp/tmp.yDKWgmjmEW ++ mktemp + local LAST_ERR=/tmp/tmp.Ddq3rIRero + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.66GBeSdQ68 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yDKWgmjmEW + cat /tmp/tmp.66GBeSdQ68 + rm /tmp/tmp.yDKWgmjmEW /tmp/tmp.66GBeSdQ68 + return 0 namespace "scheduled-backup-18460" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8dWwMg7tOQ namespace "pxc-operator" deleted + cat /tmp/tmp.Ddq3rIRero + rm /tmp/tmp.8dWwMg7tOQ /tmp/tmp.Ddq3rIRero + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'namespace/pxc-operator - ' namespace/pxc-operator - + set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.11O51lCaxT ++ mktemp + local LAST_ERR=/tmp/tmp.tJq5Zf2SaY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.11O51lCaxT namespace/pxc-operator created + cat /tmp/tmp.tJq5Zf2SaY + rm /tmp/tmp.11O51lCaxT /tmp/tmp.tJq5Zf2SaY + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.P1XW5uEBjQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.NJWJl7zEWI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.P1XW5uEBjQ ++ cat /tmp/tmp.NJWJl7zEWI ++ rm /tmp/tmp.P1XW5uEBjQ /tmp/tmp.NJWJl7zEWI ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1707-bb49e0ac-3-cluster4 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.Sw00RsS5oc ++ mktemp + local LAST_ERR=/tmp/tmp.qgtjphxVlM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1707-bb49e0ac-3-cluster4 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Sw00RsS5oc Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1707-bb49e0ac-3-cluster4" modified. + cat /tmp/tmp.qgtjphxVlM + rm /tmp/tmp.Sw00RsS5oc /tmp/tmp.qgtjphxVlM + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.wZezJWbCZt ++ mktemp + local LAST_ERR=/tmp/tmp.1nQ6jCo4BQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wZezJWbCZt customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.1nQ6jCo4BQ + rm /tmp/tmp.wZezJWbCZt /tmp/tmp.1nQ6jCo4BQ + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.FOqOgQli4m ++ mktemp + local LAST_ERR=/tmp/tmp.uICLyf8ms2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FOqOgQli4m clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.uICLyf8ms2 + rm /tmp/tmp.FOqOgQli4m /tmp/tmp.uICLyf8ms2 + return 0 + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1707-bb49e0ac^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/deploy/cw-operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.gAOxfCZ2HG ++ mktemp + local LAST_ERR=/tmp/tmp.1KwMBmB408 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gAOxfCZ2HG deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.1KwMBmB408 + rm /tmp/tmp.gAOxfCZ2HG /tmp/tmp.1KwMBmB408 + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.RtgyOtDRIc ++ mktemp + local LAST_ERR=/tmp/tmp.6ufcnllSq1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RtgyOtDRIc pod/percona-xtradb-cluster-operator-bdf674667-6gvhh condition met + cat /tmp/tmp.6ufcnllSq1 + rm /tmp/tmp.RtgyOtDRIc /tmp/tmp.6ufcnllSq1 + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.wY0GAqprEb +++ mktemp ++ local LAST_ERR=/tmp/tmp.pRzPb21GAu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wY0GAqprEb ++ cat /tmp/tmp.pRzPb21GAu ++ rm /tmp/tmp.wY0GAqprEb /tmp/tmp.pRzPb21GAu ++ return 0 + wait_pod percona-xtradb-cluster-operator-bdf674667-6gvhh 480 pxc-operator + local pod=percona-xtradb-cluster-operator-bdf674667-6gvhh + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-bdf674667-6gvhh ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-bdf674667-6gvhh condition met percona-xtradb-cluster-operator-bdf674667-6gvhh.Ok + sleep 3 + create_namespace scheduled-backup-19207 + local namespace=scheduled-backup-19207 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces scheduled-backup-19207' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces scheduled-backup-19207 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace scheduled-backup-19207 ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.E5u9SK3E3Q + local LAST_OUT=/tmp/tmp.69rmqxAH1G ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.yjpvGNAF0H + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.aOZo1LJQ7g + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-19207 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-19207 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.E5u9SK3E3Q + cat /tmp/tmp.yjpvGNAF0H + rm /tmp/tmp.E5u9SK3E3Q /tmp/tmp.yjpvGNAF0H + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scheduled-backup-19207 Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.69rmqxAH1G + cat /tmp/tmp.aOZo1LJQ7g Error from server (NotFound): namespaces "scheduled-backup-19207" not found + rm /tmp/tmp.69rmqxAH1G /tmp/tmp.aOZo1LJQ7g + return 1 + : + wait_for_delete namespace/scheduled-backup-19207 + local res=namespace/scheduled-backup-19207 + echo -n 'namespace/scheduled-backup-19207 - ' namespace/scheduled-backup-19207 - + set +o xtrace Error from server (NotFound): namespaces "scheduled-backup-19207" not found + desc 'create namespace scheduled-backup-19207' + set +o xtrace ----------------------------------------------------------------------------------- create namespace scheduled-backup-19207 ----------------------------------------------------------------------------------- + kubectl_bin create namespace scheduled-backup-19207 ++ mktemp + local LAST_OUT=/tmp/tmp.3Rdiybnmrn ++ mktemp + local LAST_ERR=/tmp/tmp.aMjA8fcN9K + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace scheduled-backup-19207 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3Rdiybnmrn namespace/scheduled-backup-19207 created + cat /tmp/tmp.aMjA8fcN9K + rm /tmp/tmp.3Rdiybnmrn /tmp/tmp.aMjA8fcN9K + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.slzVr3yDdz +++ mktemp ++ local LAST_ERR=/tmp/tmp.S2BW7TV2Ga ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.slzVr3yDdz ++ cat /tmp/tmp.S2BW7TV2Ga ++ rm /tmp/tmp.slzVr3yDdz /tmp/tmp.S2BW7TV2Ga ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1707-bb49e0ac-3-cluster4 --namespace=scheduled-backup-19207 ++ mktemp + local LAST_OUT=/tmp/tmp.dMMieYNkBL ++ mktemp + local LAST_ERR=/tmp/tmp.ZhLKLwrbob + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1707-bb49e0ac-3-cluster4 --namespace=scheduled-backup-19207 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dMMieYNkBL Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1707-bb49e0ac-3-cluster4" modified. + cat /tmp/tmp.ZhLKLwrbob + rm /tmp/tmp.dMMieYNkBL /tmp/tmp.ZhLKLwrbob + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.eJ6VVe8UFI ++ mktemp + local LAST_ERR=/tmp/tmp.FrOo7hbaaB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eJ6VVe8UFI secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.FrOo7hbaaB + rm /tmp/tmp.eJ6VVe8UFI /tmp/tmp.FrOo7hbaaB + return 0 + start_minio + deploy_helm scheduled-backup-19207 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "minio" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Tue May 14 18:12:30 2024 NAMESPACE: scheduled-backup-19207 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.scheduled-backup-19207.svc.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-19207 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-19207 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-19207 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-19207 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JjNooGnekw +++ mktemp ++ local LAST_ERR=/tmp/tmp.BcJ1NUtthO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JjNooGnekw ++ cat /tmp/tmp.BcJ1NUtthO ++ rm /tmp/tmp.JjNooGnekw /tmp/tmp.BcJ1NUtthO ++ return 0 + MINIO_POD=minio-service-76ffcfd45-lwdrl + wait_pod minio-service-76ffcfd45-lwdrl + local pod=minio-service-76ffcfd45-lwdrl + local max_retry=480 + local ns= ++ echo minio-service-76ffcfd45-lwdrl ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minio-service-76ffcfd45-lwdrl condition met minio-service-76ffcfd45-lwdrl.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.PJNbDbAUno ++ mktemp + local LAST_ERR=/tmp/tmp.c573SJ78D4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PJNbDbAUno make_bucket: operator-testing pod "aws-cli" deleted + cat /tmp/tmp.c573SJ78D4 + rm /tmp/tmp.PJNbDbAUno /tmp/tmp.c573SJ78D4 + return 0 + cluster=scheduled-backup + cat - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.fpGKNehmmm ++ mktemp + local LAST_ERR=/tmp/tmp.UM1qFB46G2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fpGKNehmmm priorityclass.scheduling.k8s.io/high-priority configured + cat /tmp/tmp.UM1qFB46G2 + rm /tmp/tmp.fpGKNehmmm /tmp/tmp.UM1qFB46G2 + return 0 + spinup_pxc scheduled-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local cluster=scheduled-backup + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.EqP6xo0PnW ++ mktemp + local LAST_ERR=/tmp/tmp.5MJygbce2z + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EqP6xo0PnW secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.5MJygbce2z + rm /tmp/tmp.EqP6xo0PnW /tmp/tmp.5MJygbce2z + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/conf/client.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/conf/client.yml ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.oGmx7Grsyg + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1707-bb49e0ac#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19207~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ mktemp + local LAST_ERR=/tmp/tmp.8kFSnjWFlc + local exit_status=0 + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ seq 0 2 + for i in '$(seq 0 2)' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oGmx7Grsyg deployment.apps/pxc-client created + cat /tmp/tmp.8kFSnjWFlc + rm /tmp/tmp.oGmx7Grsyg /tmp/tmp.8kFSnjWFlc + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-init.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1707-bb49e0ac#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_OUT=/tmp/tmp.aZJYvKfolC + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.4ONZNjeUOJ + local exit_status=0 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19207~ ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.aZJYvKfolC perconaxtradbcluster.pxc.percona.com/scheduled-backup created + cat /tmp/tmp.4ONZNjeUOJ + rm /tmp/tmp.aZJYvKfolC /tmp/tmp.4ONZNjeUOJ + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy scheduled-backup ++ local target_cluster=scheduled-backup +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.swWvSOuCN0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OjBVEEXgyC +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.swWvSOuCN0 +++ cat /tmp/tmp.OjBVEEXgyC +++ rm /tmp/tmp.swWvSOuCN0 /tmp/tmp.OjBVEEXgyC +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fXg2XDE85h ++++ mktemp +++ local LAST_ERR=/tmp/tmp.IZEKR04Z8O +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scheduled-backup -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.fXg2XDE85h +++ cat /tmp/tmp.IZEKR04Z8O +++ rm /tmp/tmp.fXg2XDE85h /tmp/tmp.IZEKR04Z8O +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scheduled-backup-proxysql ++ return + local proxy=scheduled-backup-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-19207 ++ mktemp + local LAST_OUT=/tmp/tmp.a42cScOC9S ++ mktemp + local LAST_ERR=/tmp/tmp.mhl44z49co + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-19207 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-19207 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scheduled-backup-19207 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.a42cScOC9S + cat /tmp/tmp.mhl44z49co error: no matching resources found + rm /tmp/tmp.a42cScOC9S /tmp/tmp.mhl44z49co + return 1 + true + wait_for_running scheduled-backup-proxysql 1 + local name=scheduled-backup-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-proxysql-0 480 + local pod=scheduled-backup-proxysql-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/scheduled-backup-proxysql-0 condition met scheduled-backup-proxysql-0.Ok + wait_for_running scheduled-backup-pxc 3 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 480 + local pod=scheduled-backup-pxc-0 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 480 + local pod=scheduled-backup-pxc-1 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 480 + local pod=scheduled-backup-pxc-2 + local max_retry=480 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scheduled-backup-proxysql -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h scheduled-backup-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pon7nxV8DH +++ mktemp ++ local LAST_ERR=/tmp/tmp.hPpW075ZNg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Pon7nxV8DH ++ cat /tmp/tmp.hPpW075ZNg ++ rm /tmp/tmp.Pon7nxV8DH /tmp/tmp.hPpW075ZNg ++ return 0 + client_pod=pxc-client-6644d8898f-g7r59 + wait_pod pxc-client-6644d8898f-g7r59 + local pod=pxc-client-6644d8898f-g7r59 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-g7r59 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-g7r59 condition met pxc-client-6644d8898f-g7r59.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scheduled-backup-proxysql -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h scheduled-backup-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Yl0zGy5xZE +++ mktemp ++ local LAST_ERR=/tmp/tmp.D8EvGy7FzU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Yl0zGy5xZE ++ cat /tmp/tmp.D8EvGy7FzU ++ rm /tmp/tmp.Yl0zGy5xZE /tmp/tmp.D8EvGy7FzU ++ return 0 + client_pod=pxc-client-6644d8898f-g7r59 + wait_pod pxc-client-6644d8898f-g7r59 + local pod=pxc-client-6644d8898f-g7r59 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-g7r59 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-g7r59 condition met pxc-client-6644d8898f-g7r59.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-0.scheduled-backup-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QGhaa6tZVI +++ mktemp ++ local LAST_ERR=/tmp/tmp.GV10U2nQCl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QGhaa6tZVI ++ cat /tmp/tmp.GV10U2nQCl ++ rm /tmp/tmp.QGhaa6tZVI /tmp/tmp.GV10U2nQCl ++ return 0 + client_pod=pxc-client-6644d8898f-g7r59 + wait_pod pxc-client-6644d8898f-g7r59 + local pod=pxc-client-6644d8898f-g7r59 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-g7r59 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-g7r59 condition met pxc-client-6644d8898f-g7r59.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.40X8K9WYhZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.40X8K9WYhZ/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-1.scheduled-backup-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DXx1RdJ0DJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.Xk6pbP6h6Y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DXx1RdJ0DJ ++ cat /tmp/tmp.Xk6pbP6h6Y ++ rm /tmp/tmp.DXx1RdJ0DJ /tmp/tmp.Xk6pbP6h6Y ++ return 0 + client_pod=pxc-client-6644d8898f-g7r59 + wait_pod pxc-client-6644d8898f-g7r59 + local pod=pxc-client-6644d8898f-g7r59 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-g7r59 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-g7r59 condition met pxc-client-6644d8898f-g7r59.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.40X8K9WYhZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.40X8K9WYhZ/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scheduled-backup-pxc-2.scheduled-backup-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OIJZHeaCvH +++ mktemp ++ local LAST_ERR=/tmp/tmp.TypWFOhcau ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OIJZHeaCvH ++ cat /tmp/tmp.TypWFOhcau ++ rm /tmp/tmp.OIJZHeaCvH /tmp/tmp.TypWFOhcau ++ return 0 + client_pod=pxc-client-6644d8898f-g7r59 + wait_pod pxc-client-6644d8898f-g7r59 + local pod=pxc-client-6644d8898f-g7r59 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-g7r59 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-g7r59 condition met pxc-client-6644d8898f-g7r59.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.40X8K9WYhZ/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.40X8K9WYhZ/select-1.sql ++ is_keyring_plugin_in_use scheduled-backup ++ local cluster=scheduled-backup ++ kubectl_bin exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Y9XB1zmCYf +++ mktemp ++ local LAST_ERR=/tmp/tmp.5rqbyF5ht1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it scheduled-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Y9XB1zmCYf ++ cat /tmp/tmp.5rqbyF5ht1 Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.Y9XB1zmCYf /tmp/tmp.5rqbyF5ht1 ++ return 0 + '[' '' ']' + sleep 20 + desc 'add backups schedule for pvc storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for pvc storage ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.OxV3ICYM1s +++ mktemp ++ local LAST_ERR=/tmp/tmp.ALiou7RevS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OxV3ICYM1s ++ cat /tmp/tmp.ALiou7RevS ++ rm /tmp/tmp.OxV3ICYM1s /tmp/tmp.ALiou7RevS ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1707-bb49e0ac-3-cluster4 --namespace=scheduled-backup-19207 ++ mktemp + local LAST_OUT=/tmp/tmp.DokE35pRgq ++ mktemp + local LAST_ERR=/tmp/tmp.akushhhar0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1707-bb49e0ac-3-cluster4 --namespace=scheduled-backup-19207 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DokE35pRgq Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1707-bb49e0ac-3-cluster4" modified. + cat /tmp/tmp.akushhhar0 + rm /tmp/tmp.DokE35pRgq /tmp/tmp.akushhhar0 + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-pvc.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19207~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_OUT=/tmp/tmp.cf1bLjU2j2 + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1707-bb49e0ac#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + local LAST_ERR=/tmp/tmp.Waiut9EEGt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cf1bLjU2j2 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.Waiut9EEGt + rm /tmp/tmp.cf1bLjU2j2 /tmp/tmp.Waiut9EEGt + return 0 + label_node ++ kubectl_bin get nodes --no-headers=true ++ grep -v master ++ head -n1 ++ awk '{print $1}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.avzeWQ9qVG +++ mktemp ++ local LAST_ERR=/tmp/tmp.RU7v6FVmBi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get nodes --no-headers=true ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.avzeWQ9qVG ++ cat /tmp/tmp.RU7v6FVmBi ++ rm /tmp/tmp.avzeWQ9qVG /tmp/tmp.RU7v6FVmBi ++ return 0 + LABELED_NODE=gke-jen-pxc-1707-bb49e0a-default-pool-bda8b124-c011 + kubectl_bin label nodes gke-jen-pxc-1707-bb49e0a-default-pool-bda8b124-c011 backupWorker=True --overwrite ++ mktemp + local LAST_OUT=/tmp/tmp.jSAzI4PGPF ++ mktemp + local LAST_ERR=/tmp/tmp.uqEDNbozZh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label nodes gke-jen-pxc-1707-bb49e0a-default-pool-bda8b124-c011 backupWorker=True --overwrite + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jSAzI4PGPF node/gke-jen-pxc-1707-bb49e0a-default-pool-bda8b124-c011 labeled + cat /tmp/tmp.uqEDNbozZh + rm /tmp/tmp.jSAzI4PGPF /tmp/tmp.uqEDNbozZh + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1707-bb49e0ac#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19207~ + local LAST_OUT=/tmp/tmp.mcbhbF3A6n + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.RzoGVw1Hgy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mcbhbF3A6n perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.RzoGVw1Hgy + rm /tmp/tmp.mcbhbF3A6n /tmp/tmp.RzoGVw1Hgy + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.xeSflVnKcE +++ mktemp ++ local LAST_ERR=/tmp/tmp.3on4FH9H7M ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xeSflVnKcE ++ cat /tmp/tmp.3on4FH9H7M ++ rm /tmp/tmp.xeSflVnKcE /tmp/tmp.3on4FH9H7M ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.O8fZYXBXw6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.g2d2luEZP8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.O8fZYXBXw6 ++ cat /tmp/tmp.g2d2luEZP8 ++ rm /tmp/tmp.O8fZYXBXw6 /tmp/tmp.g2d2luEZP8 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.9pOdV8nhBQ ++ mktemp + local LAST_ERR=/tmp/tmp.XnRZNhi0WV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9pOdV8nhBQ NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-202451418200-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202451418200-q6fav Running 12s + cat /tmp/tmp.XnRZNhi0WV + rm /tmp/tmp.9pOdV8nhBQ /tmp/tmp.XnRZNhi0WV + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.LJUY5pzLb3 ++ mktemp + local LAST_ERR=/tmp/tmp.RcADk7R3QR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LJUY5pzLb3 NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-lwdrl 1/1 Running 0 7m39s pxc-client-6644d8898f-g7r59 1/1 Running 0 6m26s scheduled-backup-proxysql-0 3/3 Running 0 6m24s scheduled-backup-proxysql-1 3/3 Running 0 6m6s scheduled-backup-pxc-0 1/1 Running 0 6m24s scheduled-backup-pxc-1 1/1 Running 0 5m10s scheduled-backup-pxc-2 1/1 Running 0 3m54s xb-cron-scheduled-backup-pvc-202451418200-q6fav-zgcpx 0/1 ContainerCreating 0 12s + cat /tmp/tmp.RcADk7R3QR + rm /tmp/tmp.LJUY5pzLb3 /tmp/tmp.RcADk7R3QR + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.8DoTvODwiS +++ mktemp ++ local LAST_ERR=/tmp/tmp.y17orBF0RJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8DoTvODwiS ++ cat /tmp/tmp.y17orBF0RJ ++ rm /tmp/tmp.8DoTvODwiS /tmp/tmp.y17orBF0RJ ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.3rnunfhUgH +++ mktemp ++ local LAST_ERR=/tmp/tmp.OyheJaI8CN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3rnunfhUgH ++ cat /tmp/tmp.OyheJaI8CN ++ rm /tmp/tmp.3rnunfhUgH /tmp/tmp.OyheJaI8CN ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.0vg5elBVum ++ mktemp + local LAST_ERR=/tmp/tmp.NcUUIxHlti + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0vg5elBVum NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-pvc-202451418200-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202451418200-q6fav Running 42s + cat /tmp/tmp.NcUUIxHlti + rm /tmp/tmp.0vg5elBVum /tmp/tmp.NcUUIxHlti + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.kXh9MCc7nl ++ mktemp + local LAST_ERR=/tmp/tmp.3GipklSnCT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kXh9MCc7nl NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-lwdrl 1/1 Running 0 8m10s pxc-client-6644d8898f-g7r59 1/1 Running 0 6m57s scheduled-backup-proxysql-0 3/3 Running 0 6m55s scheduled-backup-proxysql-1 3/3 Running 0 6m37s scheduled-backup-pxc-0 1/1 Running 0 6m55s scheduled-backup-pxc-1 1/1 Running 0 5m41s scheduled-backup-pxc-2 1/1 Running 0 4m25s xb-cron-scheduled-backup-pvc-202451418200-q6fav-zgcpx 1/1 Running 0 43s + cat /tmp/tmp.3GipklSnCT + rm /tmp/tmp.kXh9MCc7nl /tmp/tmp.3GipklSnCT + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.pCOQjK5MrR +++ mktemp ++ local LAST_ERR=/tmp/tmp.wfFdIPuilF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pCOQjK5MrR ++ cat /tmp/tmp.wfFdIPuilF ++ rm /tmp/tmp.pCOQjK5MrR /tmp/tmp.wfFdIPuilF ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.b7NKafuUoH +++ mktemp ++ local LAST_ERR=/tmp/tmp.5hW8POw5pw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.b7NKafuUoH ++ cat /tmp/tmp.5hW8POw5pw ++ rm /tmp/tmp.b7NKafuUoH /tmp/tmp.5hW8POw5pw ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for aws s3 storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for aws s3 storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-aws.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1707-bb49e0ac#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19207~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_OUT=/tmp/tmp.kT0n0YBSrj ++ mktemp + local LAST_ERR=/tmp/tmp.O4xfgYZCaT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kT0n0YBSrj perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.O4xfgYZCaT + rm /tmp/tmp.kT0n0YBSrj /tmp/tmp.O4xfgYZCaT + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_OUT=/tmp/tmp.xostlDhYp9 + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1707-bb49e0ac#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19207~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_ERR=/tmp/tmp.93nSzuCG6y + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xostlDhYp9 perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.93nSzuCG6y + rm /tmp/tmp.xostlDhYp9 /tmp/tmp.93nSzuCG6y + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.o1oqf7p1oc +++ mktemp ++ local LAST_ERR=/tmp/tmp.xvSuoxrAks ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.o1oqf7p1oc ++ cat /tmp/tmp.xvSuoxrAks ++ rm /tmp/tmp.o1oqf7p1oc /tmp/tmp.xvSuoxrAks ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Y9ARSA55AZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.H84mYzG002 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Y9ARSA55AZ ++ cat /tmp/tmp.H84mYzG002 ++ rm /tmp/tmp.Y9ARSA55AZ /tmp/tmp.H84mYzG002 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.lYCq1XQZvU ++ mktemp + local LAST_ERR=/tmp/tmp.wn2Y0Qpdyl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lYCq1XQZvU NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-202451418226-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2024-05-14-18:22:06-full Running 10s cron-scheduled-backup-pvc-202451418200-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202451418200-q6fav Succeeded 90s 2m16s + cat /tmp/tmp.wn2Y0Qpdyl + rm /tmp/tmp.lYCq1XQZvU /tmp/tmp.wn2Y0Qpdyl + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.zO8vhntHB2 ++ mktemp + local LAST_ERR=/tmp/tmp.qev2QXc6SN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zO8vhntHB2 NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-lwdrl 1/1 Running 0 9m44s pxc-client-6644d8898f-g7r59 1/1 Running 0 8m31s scheduled-backup-proxysql-0 3/3 Running 0 8m29s scheduled-backup-proxysql-1 3/3 Running 0 8m11s scheduled-backup-pxc-0 1/1 Running 0 8m29s scheduled-backup-pxc-1 1/1 Running 0 7m15s scheduled-backup-pxc-2 1/1 Running 0 5m59s xb-cron-scheduled-backup-aws-s3-202451418226-q6fav-gqv4r 1/1 Running 0 11s xb-cron-scheduled-backup-pvc-202451418200-q6fav-zgcpx 0/1 Completed 0 2m17s + cat /tmp/tmp.qev2QXc6SN + rm /tmp/tmp.zO8vhntHB2 /tmp/tmp.qev2QXc6SN + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.seSk8C8hCK +++ mktemp ++ local LAST_ERR=/tmp/tmp.vPAGGv1JjF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.seSk8C8hCK ++ cat /tmp/tmp.vPAGGv1JjF ++ rm /tmp/tmp.seSk8C8hCK /tmp/tmp.vPAGGv1JjF ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.OXhxnXK46N +++ mktemp ++ local LAST_ERR=/tmp/tmp.UAps9wWH7q ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OXhxnXK46N ++ cat /tmp/tmp.UAps9wWH7q ++ rm /tmp/tmp.OXhxnXK46N /tmp/tmp.UAps9wWH7q ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.VwnrT8Q1bj ++ mktemp + local LAST_ERR=/tmp/tmp.HK1rWXHEl7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VwnrT8Q1bj NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-202451418226-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2024-05-14-18:22:06-full Succeeded 7s 40s cron-scheduled-backup-pvc-202451418200-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202451418200-q6fav Succeeded 2m 2m46s + cat /tmp/tmp.HK1rWXHEl7 + rm /tmp/tmp.VwnrT8Q1bj /tmp/tmp.HK1rWXHEl7 + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.AAexv5NYJe ++ mktemp + local LAST_ERR=/tmp/tmp.p8AMFhTJ36 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AAexv5NYJe NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-lwdrl 1/1 Running 0 10m pxc-client-6644d8898f-g7r59 1/1 Running 0 9m scheduled-backup-proxysql-0 3/3 Running 0 8m58s scheduled-backup-proxysql-1 3/3 Running 0 8m40s scheduled-backup-pxc-0 1/1 Running 0 8m58s scheduled-backup-pxc-1 1/1 Running 0 7m44s scheduled-backup-pxc-2 1/1 Running 0 6m28s xb-cron-scheduled-backup-aws-s3-202451418226-q6fav-gqv4r 0/1 Completed 0 40s xb-cron-scheduled-backup-pvc-202451418200-q6fav-zgcpx 0/1 Completed 0 2m46s + cat /tmp/tmp.p8AMFhTJ36 + rm /tmp/tmp.AAexv5NYJe /tmp/tmp.p8AMFhTJ36 + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.F4kUQYP95X +++ mktemp ++ local LAST_ERR=/tmp/tmp.k8ztwCsGQH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.F4kUQYP95X ++ cat /tmp/tmp.k8ztwCsGQH ++ rm /tmp/tmp.F4kUQYP95X /tmp/tmp.k8ztwCsGQH ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.lb6kxn2dAc +++ mktemp ++ local LAST_ERR=/tmp/tmp.eRX5KIZ0OQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lb6kxn2dAc ++ cat /tmp/tmp.eRX5KIZ0OQ ++ rm /tmp/tmp.lb6kxn2dAc /tmp/tmp.eRX5KIZ0OQ ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for minio storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for minio storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-minio.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1707-bb49e0ac#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_OUT=/tmp/tmp.8Im5XI51Pp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + local LAST_ERR=/tmp/tmp.MgzTFWxkX8 + local exit_status=0 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19207~ ++ seq 0 2 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8Im5XI51Pp perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.MgzTFWxkX8 + rm /tmp/tmp.8Im5XI51Pp /tmp/tmp.MgzTFWxkX8 + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.j8xDBj70cV + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1707-bb49e0ac#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19207~ + local LAST_ERR=/tmp/tmp.oVTR0LIdo0 + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.j8xDBj70cV perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.oVTR0LIdo0 + rm /tmp/tmp.j8xDBj70cV /tmp/tmp.oVTR0LIdo0 + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.r0s8QiC9c0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.osYQ9Do3hd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.r0s8QiC9c0 ++ cat /tmp/tmp.osYQ9Do3hd ++ rm /tmp/tmp.r0s8QiC9c0 /tmp/tmp.osYQ9Do3hd ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.Me0xKPjgHE +++ mktemp ++ local LAST_ERR=/tmp/tmp.x6P7Zem5w1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Me0xKPjgHE ++ cat /tmp/tmp.x6P7Zem5w1 ++ rm /tmp/tmp.Me0xKPjgHE /tmp/tmp.x6P7Zem5w1 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.cnykRE58GB ++ mktemp + local LAST_ERR=/tmp/tmp.eKWcf9R131 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cnykRE58GB NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-202451418226-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2024-05-14-18:22:06-full Succeeded 103s 2m16s cron-scheduled-backup-minio-2024514182410-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2024-05-14-18:24:10-full Running 12s cron-scheduled-backup-pvc-202451418200-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202451418200-q6fav Succeeded 3m36s 4m22s + cat /tmp/tmp.eKWcf9R131 + rm /tmp/tmp.cnykRE58GB /tmp/tmp.eKWcf9R131 + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.AA9rf8Miio ++ mktemp + local LAST_ERR=/tmp/tmp.yhFuzzBtcm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AA9rf8Miio NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-lwdrl 1/1 Running 0 11m pxc-client-6644d8898f-g7r59 1/1 Running 0 10m scheduled-backup-proxysql-0 3/3 Running 0 10m scheduled-backup-proxysql-1 3/3 Running 0 10m scheduled-backup-pxc-0 1/1 Running 0 10m scheduled-backup-pxc-1 1/1 Running 0 9m21s scheduled-backup-pxc-2 1/1 Running 0 8m5s xb-cron-scheduled-backup-aws-s3-202451418226-q6fav-gqv4r 0/1 Completed 0 2m17s xb-cron-scheduled-backup-minio-2024514182410-q6fav-d2slc 1/1 Running 0 13s xb-cron-scheduled-backup-pvc-202451418200-q6fav-zgcpx 0/1 Completed 0 4m23s + cat /tmp/tmp.yhFuzzBtcm + rm /tmp/tmp.AA9rf8Miio /tmp/tmp.yhFuzzBtcm + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.eM6abbhMht +++ mktemp ++ local LAST_ERR=/tmp/tmp.XXLp7XooSg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eM6abbhMht ++ cat /tmp/tmp.XXLp7XooSg ++ rm /tmp/tmp.eM6abbhMht /tmp/tmp.XXLp7XooSg ++ return 0 + [[ 0 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.8hcZz97Dqs +++ mktemp ++ local LAST_ERR=/tmp/tmp.s79kBdGvM0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8hcZz97Dqs ++ cat /tmp/tmp.s79kBdGvM0 ++ rm /tmp/tmp.8hcZz97Dqs /tmp/tmp.s79kBdGvM0 ++ return 0 + [[ 0 -gt 0 ]] + desc 'add backups schedule for gcs storage' + set +o xtrace ----------------------------------------------------------------------------------- add backups schedule for gcs storage ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-gcs.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19207~ + local LAST_OUT=/tmp/tmp.EcGWtSiycY + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1707-bb49e0ac#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.5gJTKMehtY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EcGWtSiycY perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.5gJTKMehtY + rm /tmp/tmp.EcGWtSiycY /tmp/tmp.5gJTKMehtY + return 0 + sleep 61 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1707/e2e-tests/scheduled-backup/conf/scheduled-backup-disable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.mDyXva4e3b + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1707-bb49e0ac#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scheduled-backup-19207~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_ERR=/tmp/tmp.7Fs4SgAb9Q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mDyXva4e3b perconaxtradbcluster.pxc.percona.com/scheduled-backup configured + cat /tmp/tmp.7Fs4SgAb9Q + rm /tmp/tmp.mDyXva4e3b /tmp/tmp.7Fs4SgAb9Q + return 0 + wait_all_backups ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.gVMqZ153py +++ mktemp ++ local LAST_ERR=/tmp/tmp.hBkGFNqvCs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gVMqZ153py ++ cat /tmp/tmp.hBkGFNqvCs ++ rm /tmp/tmp.gVMqZ153py /tmp/tmp.hBkGFNqvCs ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ wc -l ++ grep :Failed +++ mktemp ++ local LAST_OUT=/tmp/tmp.6XLN7Y9WvT +++ mktemp ++ local LAST_ERR=/tmp/tmp.bQ9Tk1ZE42 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6XLN7Y9WvT ++ cat /tmp/tmp.bQ9Tk1ZE42 ++ rm /tmp/tmp.6XLN7Y9WvT /tmp/tmp.bQ9Tk1ZE42 ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.xA06H6xXUt ++ mktemp + local LAST_ERR=/tmp/tmp.DjxFhFW89q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xA06H6xXUt NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-202451418226-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2024-05-14-18:22:06-full Succeeded 3m20s 3m53s cron-scheduled-backup-gcp-cs-2024514182546-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2024-05-14-18:25:46-full Running 13s cron-scheduled-backup-minio-2024514182410-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2024-05-14-18:24:10-full Succeeded 84s 109s cron-scheduled-backup-pvc-202451418200-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202451418200-q6fav Succeeded 5m13s 5m59s + cat /tmp/tmp.DjxFhFW89q + rm /tmp/tmp.xA06H6xXUt /tmp/tmp.DjxFhFW89q + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.YYtxQXWZhf ++ mktemp + local LAST_ERR=/tmp/tmp.ksiL8mFJKj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YYtxQXWZhf NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-lwdrl 1/1 Running 0 13m pxc-client-6644d8898f-g7r59 1/1 Running 0 12m scheduled-backup-proxysql-0 3/3 Running 0 12m scheduled-backup-proxysql-1 3/3 Running 0 11m scheduled-backup-pxc-0 1/1 Running 0 12m scheduled-backup-pxc-1 1/1 Running 0 10m scheduled-backup-pxc-2 1/1 Running 0 9m41s xb-cron-scheduled-backup-aws-s3-202451418226-q6fav-gqv4r 0/1 Completed 0 3m53s xb-cron-scheduled-backup-gcp-cs-2024514182546-q6fav-c4skk 1/1 Running 0 13s xb-cron-scheduled-backup-minio-2024514182410-q6fav-d2slc 0/1 Completed 0 109s xb-cron-scheduled-backup-pvc-202451418200-q6fav-zgcpx 0/1 Completed 0 5m59s + cat /tmp/tmp.ksiL8mFJKj + rm /tmp/tmp.YYtxQXWZhf /tmp/tmp.ksiL8mFJKj + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.1fku6zhVxn +++ mktemp ++ local LAST_ERR=/tmp/tmp.jwGZ2UNpK7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1fku6zhVxn ++ cat /tmp/tmp.jwGZ2UNpK7 ++ rm /tmp/tmp.1fku6zhVxn /tmp/tmp.jwGZ2UNpK7 ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.7sDQxrLpkv +++ mktemp ++ local LAST_ERR=/tmp/tmp.cCBBZyk6eE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7sDQxrLpkv ++ cat /tmp/tmp.cCBBZyk6eE ++ rm /tmp/tmp.7sDQxrLpkv /tmp/tmp.cCBBZyk6eE ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-0 condition met scheduled-backup-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-1 1 + local pod=scheduled-backup-pxc-1 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-1 condition met scheduled-backup-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-2 1 + local pod=scheduled-backup-pxc-2 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scheduled-backup-pxc-2 condition met scheduled-backup-pxc-2.Ok + echo + kubectl_bin get pxc-backup ++ mktemp + local LAST_OUT=/tmp/tmp.fYUlxIBQuy ++ mktemp + local LAST_ERR=/tmp/tmp.SR9q9GISVZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pxc-backup + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fYUlxIBQuy NAME CLUSTER STORAGE DESTINATION STATUS COMPLETED AGE cron-scheduled-backup-aws-s3-202451418226-q6fav scheduled-backup aws-s3 s3://operator-testing/scheduled-backup-2024-05-14-18:22:06-full Succeeded 3m49s 4m22s cron-scheduled-backup-gcp-cs-2024514182546-q6fav scheduled-backup gcp-cs s3://operator-testing/scheduled-backup-2024-05-14-18:25:46-full Running 42s cron-scheduled-backup-minio-2024514182410-q6fav scheduled-backup minio s3://operator-testing/scheduled-backup-2024-05-14-18:24:10-full Succeeded 113s 2m18s cron-scheduled-backup-pvc-202451418200-q6fav scheduled-backup pvc pvc/xb-cron-scheduled-backup-pvc-202451418200-q6fav Succeeded 5m42s 6m28s + cat /tmp/tmp.SR9q9GISVZ + rm /tmp/tmp.fYUlxIBQuy /tmp/tmp.SR9q9GISVZ + return 0 + echo + kubectl_bin get pods ++ mktemp + local LAST_OUT=/tmp/tmp.eD8u7U902o ++ mktemp + local LAST_ERR=/tmp/tmp.1MyNePj1fX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get pods + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eD8u7U902o NAME READY STATUS RESTARTS AGE minio-service-76ffcfd45-lwdrl 1/1 Running 0 13m pxc-client-6644d8898f-g7r59 1/1 Running 0 12m scheduled-backup-proxysql-0 3/3 Running 0 12m scheduled-backup-proxysql-1 0/3 Init:0/1 0 18s scheduled-backup-pxc-0 0/1 Pending 0 4s scheduled-backup-pxc-1 1/1 Running 0 11m scheduled-backup-pxc-2 1/1 Running 0 10m xb-cron-scheduled-backup-aws-s3-202451418226-q6fav-gqv4r 0/1 Completed 0 4m22s xb-cron-scheduled-backup-gcp-cs-2024514182546-q6fav-c4skk 0/1 Completed 0 42s xb-cron-scheduled-backup-minio-2024514182410-q6fav-d2slc 0/1 Completed 0 2m18s xb-cron-scheduled-backup-pvc-202451418200-q6fav-zgcpx 0/1 Completed 0 6m28s + cat /tmp/tmp.1MyNePj1fX + rm /tmp/tmp.eD8u7U902o /tmp/tmp.1MyNePj1fX + return 0 + sleep 20 ++ get_running_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep -vE ':Succeeded|:Failed' ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.4GQEOJ4UGU +++ mktemp ++ local LAST_ERR=/tmp/tmp.yBtDh8xMFN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4GQEOJ4UGU ++ cat /tmp/tmp.yBtDh8xMFN ++ rm /tmp/tmp.4GQEOJ4UGU /tmp/tmp.yBtDh8xMFN ++ return 0 + [[ 1 -ne 0 ]] ++ get_failed_backups_amount ++ kubectl_bin get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ grep :Failed ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.s4HO0XkkTV +++ mktemp ++ local LAST_ERR=/tmp/tmp.DSypP2RBHe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc-backup '-o=jsonpath={range .items[*]}{.metadata.name}{":"}{.spec.storageName}{":"}{.status.state}{"\n"}{end}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.s4HO0XkkTV ++ cat /tmp/tmp.DSypP2RBHe ++ rm /tmp/tmp.s4HO0XkkTV /tmp/tmp.DSypP2RBHe ++ return 0 + [[ 0 -eq 0 ]] + wait_for_running scheduled-backup-pxc 3 1 + local name=scheduled-backup-pxc + let last_pod=2 + local max_retry=1 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scheduled-backup-pxc-0 1 + local pod=scheduled-backup-pxc-0 + local max_retry=1 + local ns= ++ echo scheduled-backup-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace error: timed out waiting for the condition on pods/scheduled-backup-pxc-0 scheduled-backup-pxc-0.Name: scheduled-backup-pxc-0 Namespace: scheduled-backup-19207 Priority: 0 Service Account: default Node: Labels: app.kubernetes.io/component=pxc app.kubernetes.io/instance=scheduled-backup app.kubernetes.io/managed-by=percona-xtradb-cluster-operator app.kubernetes.io/name=percona-xtradb-cluster app.kubernetes.io/part-of=percona-xtradb-cluster controller-revision-hash=scheduled-backup-pxc-7888dff66c statefulset.kubernetes.io/pod-name=scheduled-backup-pxc-0 Annotations: cloud.google.com/cluster_autoscaler_unhelpable_since: 2024-05-14T18:26:26+0000 cloud.google.com/cluster_autoscaler_unhelpable_until: Inf percona.com/configuration-hash: d41d8cd98f00b204e9800998ecf8427e percona.com/ssl-hash: cfd6a52398268173b51d7cdb1331c09a percona.com/ssl-internal-hash: 751e59a1c1d94ac67f9d6bb6e756acdc Status: Pending IP: IPs: Controlled By: StatefulSet/scheduled-backup-pxc Init Containers: pxc-init: Image: perconalab/percona-xtradb-cluster-operator:PR-1707-bb49e0ac Port: Host Port: Command: /pxc-init-entrypoint.sh Limits: cpu: 50m memory: 50M Requests: cpu: 50m memory: 50M Environment: Mounts: /var/lib/mysql from datadir (rw) /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-zdgtj (ro) Containers: pxc: Image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0 Ports: 3306/TCP, 4444/TCP, 4567/TCP, 4568/TCP, 33062/TCP, 33060/TCP Host Ports: 0/TCP, 0/TCP, 0/TCP, 0/TCP, 0/TCP, 0/TCP Command: /var/lib/mysql/pxc-entrypoint.sh Args: mysqld Liveness: exec [/var/lib/mysql/liveness-check.sh] delay=300s timeout=5s period=10s #success=1 #failure=3 Readiness: exec [/var/lib/mysql/readiness-check.sh] delay=15s timeout=15s period=30s #success=1 #failure=5 Environment Variables from: scheduled-backup-env-vars-pxc Secret Optional: true Environment: PXC_SERVICE: scheduled-backup-pxc-unready MONITOR_HOST: % MYSQL_ROOT_PASSWORD: Optional: false XTRABACKUP_PASSWORD: Optional: false MONITOR_PASSWORD: Optional: false CLUSTER_HASH: 3895616 OPERATOR_ADMIN_PASSWORD: Optional: false LIVENESS_CHECK_TIMEOUT: 5 READINESS_CHECK_TIMEOUT: 15 DEFAULT_AUTHENTICATION_PLUGIN: mysql_native_password Mounts: /etc/my.cnf.d from auto-config (rw) /etc/mysql/init-file from mysql-init-file (rw) /etc/mysql/mysql-users-secret from mysql-users-secret-file (rw) /etc/mysql/ssl from ssl (rw) /etc/mysql/ssl-internal from ssl-internal (rw) /etc/mysql/vault-keyring-secret from vault-keyring-secret (rw) /etc/percona-xtradb-cluster.conf.d from config (rw) /tmp from tmp (rw) /var/lib/mysql from datadir (rw) /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-zdgtj (ro) Conditions: Type Status PodScheduled False Volumes: datadir: Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace) ClaimName: datadir-scheduled-backup-pxc-0 ReadOnly: false tmp: Type: EmptyDir (a temporary directory that shares a pod's lifetime) Medium: SizeLimit: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: scheduled-backup-pxc Optional: true ssl-internal: Type: Secret (a volume populated by a Secret) SecretName: some-name-ssl-internal Optional: true ssl: Type: Secret (a volume populated by a Secret) SecretName: some-name-ssl Optional: false auto-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: auto-scheduled-backup-pxc Optional: true vault-keyring-secret: Type: Secret (a volume populated by a Secret) SecretName: scheduled-backup-vault Optional: true mysql-users-secret-file: Type: Secret (a volume populated by a Secret) SecretName: internal-scheduled-backup Optional: false mysql-init-file: Type: Secret (a volume populated by a Secret) SecretName: scheduled-backup-mysql-init Optional: true kube-api-access-zdgtj: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal NotTriggerScaleUp 30s cluster-autoscaler pod didn't trigger scale-up: Warning FailedScheduling 28s (x3 over 32s) default-scheduler 0/3 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }, 2 node(s) didn't match pod anti-affinity rules. preemption: 0/3 nodes are available: 1 Preemption is not helpful for scheduling, 2 No preemption victims found for incoming pod.. Defaulted container "pxc" out of: pxc, pxc-init (init) 2024-05-14T18:12:02.997Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.26.15-gke.1300000"} 2024-05-14T18:12:02.997Z INFO setup Manager starting up {"gitCommit": "bb49e0ace4c0c85d9e6eb6fc7243c481ddec0d0a", "gitBranch": "PR-1707-bb49e0ac", "buildTime": "2024-05-14T16:17:41Z", "goVersion": "go1.22.3", "os": "linux", "arch": "amd64"} 2024-05-14T18:12:02.998Z INFO setup Registering Components. 2024-05-14T18:12:05.577Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2024-05-14T18:12:05.580Z INFO setup Starting the Cmd. 2024-05-14T18:12:05.580Z INFO controller-runtime.metrics Starting metrics server 2024-05-14T18:12:05.580Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2024-05-14T18:12:05.580Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2024-05-14T18:12:05.580Z INFO controller-runtime.webhook Starting webhook server 2024-05-14T18:12:05.581Z INFO controller-runtime.certwatcher Updated current TLS certificate 2024-05-14T18:12:05.581Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2024-05-14T18:12:05.581Z INFO controller-runtime.certwatcher Starting certificate watcher 2024-05-14T18:12:05.681Z INFO attempting to acquire leader lease pxc-operator/08db1feb.percona.com... 2024-05-14T18:12:05.698Z INFO successfully acquired lease pxc-operator/08db1feb.percona.com 2024-05-14T18:12:05.698Z DEBUG events percona-xtradb-cluster-operator-bdf674667-6gvhh_0cc79051-7260-4c55-80c2-585adf609a6d became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"105b9846-30e2-4789-8142-233815fba299","apiVersion":"coordination.k8s.io/v1","resourceVersion":"60582"}, "reason": "LeaderElection"} 2024-05-14T18:12:05.699Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: unknown type"} 2024-05-14T18:12:05.699Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: unknown type"} 2024-05-14T18:12:05.699Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: unknown type"} 2024-05-14T18:12:05.699Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2024-05-14T18:12:05.699Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2024-05-14T18:12:05.699Z INFO Starting Controller {"controller": "pxc-controller"} 2024-05-14T18:12:05.908Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2024-05-14T18:12:05.908Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2024-05-14T18:12:05.908Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2024-05-14T18:13:47.929Z INFO Set CR version {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "11ddb6da-cda7-4d46-85f1-4df82a171613", "version": "1.15.0"} 2024-05-14T18:13:48.810Z ERROR Reconciler error {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "11ddb6da-cda7-4d46-85f1-4df82a171613", "error": "ProxySQL upgrade error: update error: Operation cannot be fulfilled on statefulsets.apps \"scheduled-backup-proxysql\": the object has been modified; please apply your changes to the latest version and try again", "errorVerbose": "Operation cannot be fulfilled on statefulsets.apps \"scheduled-backup-proxysql\": the object has been modified; please apply your changes to the latest version and try again\nupdate error\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:221\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:375\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nProxySQL upgrade error\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:377\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:13:49.012Z ERROR Reconciler error {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "1a5a261d-21d7-43fb-b594-27f4fe75d199", "error": "create newStatefulSetNode: Operation cannot be fulfilled on statefulsets.apps \"scheduled-backup-pxc\": the object has been modified; please apply your changes to the latest version and try again", "errorVerbose": "Operation cannot be fulfilled on statefulsets.apps \"scheduled-backup-pxc\": the object has been modified; please apply your changes to the latest version and try again\ncreate newStatefulSetNode\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:649\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:302\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:15:06.844Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4ba14503-f7c9-4de1-ae12-e950032ea1ec", "user": "operator"} 2024-05-14T18:15:06.888Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4ba14503-f7c9-4de1-ae12-e950032ea1ec", "user": "monitor"} 2024-05-14T18:15:06.998Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4ba14503-f7c9-4de1-ae12-e950032ea1ec"} 2024-05-14T18:15:07.045Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4ba14503-f7c9-4de1-ae12-e950032ea1ec"} 2024-05-14T18:15:07.144Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4ba14503-f7c9-4de1-ae12-e950032ea1ec", "user": "xtrabackup"} 2024-05-14T18:15:07.252Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4ba14503-f7c9-4de1-ae12-e950032ea1ec"} 2024-05-14T18:15:07.303Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4ba14503-f7c9-4de1-ae12-e950032ea1ec", "user": "replication"} 2024-05-14T18:15:07.419Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4ba14503-f7c9-4de1-ae12-e950032ea1ec", "err": "get primary pxc pod: not found"} 2024-05-14T18:15:11.906Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "d11ffe04-a680-4ec8-87c0-b37871fa6572", "err": "get primary pxc pod: not found"} 2024-05-14T18:15:17.180Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4f499cee-35c3-4816-8e50-1dc5c2a591ae", "err": "get primary pxc pod: not found"} 2024-05-14T18:15:23.028Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "16260444-4216-41fd-b6fd-f824e42d932a", "err": "get primary pxc pod: not found"} 2024-05-14T18:17:35.547Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "b3f73e0c-75d5-42ae-883c-22cf3b0f4a5e", "user": "root"} 2024-05-14T18:17:35.872Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "b3f73e0c-75d5-42ae-883c-22cf3b0f4a5e", "new version": "8.0.36-28.1"} 2024-05-14T18:17:37.981Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "b3f73e0c-75d5-42ae-883c-22cf3b0f4a5e"} 2024-05-14T18:17:42.830Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "9746cee6-3682-4dba-ac04-eba6c3b31b40"} 2024-05-14T18:17:48.177Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "6f35cd3b-fd23-48f9-b228-4c02ed97a3b2"} 2024-05-14T18:17:54.188Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "e1fa37af-de93-48d2-bd61-71a73d8be8a7"} 2024-05-14T18:17:58.893Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4709421e-3bab-4986-a1e4-3d2a21a5a25b"} 2024-05-14T18:18:04.281Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "838c9319-fc36-4f80-b074-bc9f7c179cdb"} 2024-05-14T18:18:09.721Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "a5eec13f-83de-4985-b7b5-5a44f753b5d3"} 2024-05-14T18:18:15.261Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "f4ca3490-0d31-4d8b-bdcf-21ea49aea625"} 2024-05-14T18:18:20.512Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "37f995d0-a00e-4eb6-be73-46f0fb5bf537"} 2024-05-14T18:18:25.912Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "a00913ec-f480-437c-90d5-102a46c6fe11"} 2024-05-14T18:18:31.303Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "bbfada60-e004-4825-9024-143453dfea98"} 2024-05-14T18:18:36.584Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "0dfd6b12-b855-4f2e-9fcd-50567df7b139"} 2024-05-14T18:18:42.320Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "593fa50d-3333-4df1-ac5a-da862e9d267f"} 2024-05-14T18:18:47.603Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "faa2a3fb-2659-488a-aae9-dc91d901356f"} 2024-05-14T18:18:52.796Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "9863cc11-4c24-4218-9ae9-5e5cdf2bb7c7"} 2024-05-14T18:18:58.188Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "e4c2452b-20c5-49ff-b838-002274135b3e"} 2024-05-14T18:19:00.135Z INFO Creating or updating backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "034a1e34-776e-4601-b740-c3a9d6f768a4", "name": "fc020-each-min-pvc", "schedule": "* * * * *"} 2024-05-14T18:19:02.384Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "034a1e34-776e-4601-b740-c3a9d6f768a4"} 2024-05-14T18:19:09.758Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "9f07dc60-3bad-4c13-861f-8f55f941e8f9"} 2024-05-14T18:19:15.190Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "60feba73-8411-4766-b3dd-f273366d7544"} 2024-05-14T18:19:20.558Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "f7ea0873-60a1-4583-9a85-a21bf70cb640"} 2024-05-14T18:19:25.959Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "ed5cb63c-dad8-4d6b-883f-67076c95ec8c"} 2024-05-14T18:19:31.253Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "3c86d4d3-6484-4238-8719-b1249fe44478"} 2024-05-14T18:19:36.637Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "f2f7d355-49b9-4beb-a66f-b2114790779f"} 2024-05-14T18:19:42.043Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "6f22bbc1-22b6-442b-8870-5c28f2d38487"} 2024-05-14T18:19:47.375Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "268a62e2-7dcd-4cc0-9f69-d1c420102102"} 2024-05-14T18:19:52.764Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "29e2ee5d-52ed-4dae-91ed-d0c696d0ec95"} 2024-05-14T18:19:58.036Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "87ec25c8-bb52-4859-9f31-b3992820aafb"} 2024-05-14T18:20:00.015Z INFO Creating a new volume for backup {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-pvc-202451418200-q6fav", "reconcileID": "51b2a905-6ddc-4c9b-9893-2ade56c79fd1", "Namespace": "scheduled-backup-19207", "Name": "xb-cron-scheduled-backup-pvc-202451418200-q6fav"} 2024-05-14T18:20:00.099Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-pvc-202451418200-q6fav", "reconcileID": "51b2a905-6ddc-4c9b-9893-2ade56c79fd1", "Namespace": "scheduled-backup-19207", "Name": "xb-cron-scheduled-backup-pvc-202451418200-q6fav"} 2024-05-14T18:20:03.562Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "9beb0cb3-6d4d-4089-9d3c-5922121d8568"} 2024-05-14T18:20:04.564Z INFO deleting outdated backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "781ee6e9-45d6-48f2-ab36-392a582ff66a", "name": "fc020-each-min-pvc"} 2024-05-14T18:20:06.775Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "781ee6e9-45d6-48f2-ab36-392a582ff66a"} 2024-05-14T18:20:14.421Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "e490edd5-ff35-4a61-803e-e58aaba650cd"} 2024-05-14T18:20:20.015Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "7ebf5ce9-8428-44eb-82e0-09f2737d6340"} 2024-05-14T18:20:25.335Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "c7da7e9f-4366-45ed-88b2-9e8a37e16407"} 2024-05-14T18:20:30.668Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "8fd611f3-5974-4cb1-aa1a-8faef46e1f71"} 2024-05-14T18:20:36.003Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "a1b234dd-9247-43f6-b4ad-fbf282101943"} 2024-05-14T18:20:41.444Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "18807c96-c470-4240-8adf-5def7b138c48"} 2024-05-14T18:20:46.866Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "47d022ba-8f8d-47b5-b787-bb6d8316c703"} 2024-05-14T18:20:52.324Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "35c11fe8-ff65-44bb-83e6-74dbbef500ec"} 2024-05-14T18:20:57.637Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "ac723813-f6bc-46b0-b0d0-a92d4aa447df"} 2024-05-14T18:21:03.051Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "6c9cea11-b679-469f-9c73-a59ba3da2d8b"} 2024-05-14T18:21:06.679Z INFO Creating or updating backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "3f0fddd9-9654-4239-a810-ab21a7016f57", "name": "fc020-each-min-aws-s3", "schedule": "* * * * *"} 2024-05-14T18:21:08.591Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "8b56999e-0166-40e3-8d52-7fa64793203b"} 2024-05-14T18:21:13.959Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "2811d9b5-ea8b-4dad-85b0-d253dff2eec9"} 2024-05-14T18:21:19.158Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "8348a665-b055-4cce-a7cd-dc9c8e7b774a"} 2024-05-14T18:21:25.146Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "7f099476-d93c-49ff-8297-dcf9eef4c867"} 2024-05-14T18:21:30.536Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4cfee104-0933-481e-8a1b-6f225bcb6f99"} 2024-05-14T18:21:35.937Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "a5cccfee-3413-4eb2-93a4-5e3fbd0d97fb"} 2024-05-14T18:21:41.386Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "9e91feb3-728a-4c22-ad0f-c2ccbd63a86d"} 2024-05-14T18:21:46.706Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4d5cf890-fbd0-4baf-8260-2e100a8e5030"} 2024-05-14T18:21:52.366Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "8e93c632-fe94-41cf-beb4-5aaabf2d094a"} 2024-05-14T18:21:57.587Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "3f83c01d-cbdd-4f09-977b-56e1bdf69324"} 2024-05-14T18:22:03.047Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "c88487ad-bf4d-4a07-8a20-73dbb4945ffe"} 2024-05-14T18:22:06.045Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-aws-s3-202451418226-q6fav", "reconcileID": "9732beb3-ff48-42f5-af57-092b9f772f89", "Namespace": "scheduled-backup-19207", "Name": "xb-cron-scheduled-backup-aws-s3-202451418226-q6fav"} 2024-05-14T18:22:08.612Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "fb2897e5-6dba-44bc-8919-d5c753351925"} 2024-05-14T18:22:09.342Z INFO deleting outdated backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "97501478-46fc-4802-8569-62c1f6c0a39b", "name": "fc020-each-min-aws-s3"} 2024-05-14T18:22:11.618Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "97501478-46fc-4802-8569-62c1f6c0a39b"} 2024-05-14T18:22:19.175Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4882b82a-b8c0-4e88-8179-a4eabee3e864"} 2024-05-14T18:22:25.190Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "1fed8c60-9554-4002-9883-a21c2dec211e"} 2024-05-14T18:22:30.682Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "27c17c44-cf84-4f7e-b37c-ef13c49b776a"} 2024-05-14T18:22:35.894Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "e3bb2bb7-93df-4b1c-bdaa-501705879ad8"} 2024-05-14T18:22:41.281Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "a3d0c597-7b6f-4fae-ae73-d086e06d6398"} 2024-05-14T18:22:46.598Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "22da8e95-9fca-41a4-8c6d-1f79a85b361a"} 2024-05-14T18:22:52.172Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "ede859a0-b8c9-45ef-80b8-ec2bc57ef4ca"} 2024-05-14T18:22:57.426Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "97b3fd0c-d910-4c3e-8467-6010e04ff177"} 2024-05-14T18:23:02.963Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "65af365b-2dbb-4134-a231-6da329e810c2"} 2024-05-14T18:23:08.270Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "825f57a6-a97a-487d-8a03-3ea20c06635f"} 2024-05-14T18:23:10.348Z INFO Creating or updating backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "0698a517-cc5b-4dc3-a0b4-f2680a3ea020", "name": "fc020-each-min-minio", "schedule": "* * * * *"} 2024-05-14T18:23:12.500Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "0698a517-cc5b-4dc3-a0b4-f2680a3ea020"} 2024-05-14T18:23:19.576Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "513aab17-3d4e-4b12-a4df-b86607eb9677"} 2024-05-14T18:23:25.361Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "7da48092-fa8c-4782-8c8b-937aad7b1ca6"} 2024-05-14T18:23:30.656Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "fb827fb9-07a4-42e9-93dd-25fc56ecf94a"} 2024-05-14T18:23:36.023Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "40dc7f89-1400-4720-8134-63dd146d9257"} 2024-05-14T18:23:41.344Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "f0326131-3270-475c-8eb2-10a9ade7a977"} 2024-05-14T18:23:46.773Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "db20d030-69bd-4ab5-b326-7e538b29a049"} 2024-05-14T18:23:52.195Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "493e248d-ee5b-4b8a-839c-6d4acd06ad3f"} 2024-05-14T18:23:57.435Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "91ca5c3d-e51d-43fd-842b-6cf9faf56ec2"} 2024-05-14T18:24:02.870Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "be7261f2-6568-400e-996c-a6aee8ba0309"} 2024-05-14T18:24:08.284Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "0095e4b7-2ba9-4d46-91bf-89f54906f9f4"} 2024-05-14T18:24:10.092Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-minio-2024514182410-q6fav", "reconcileID": "445d56ec-4d48-41f0-820d-0c613fbf8487", "Namespace": "scheduled-backup-19207", "Name": "xb-cron-scheduled-backup-minio-2024514182410-q6fav"} 2024-05-14T18:24:13.067Z INFO deleting outdated backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "2096b882-e127-4268-bc07-e16120fcd1db", "name": "fc020-each-min-minio"} 2024-05-14T18:24:13.706Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "c6c91dfb-9c04-45be-8502-b728e473f9bb"} 2024-05-14T18:24:18.964Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "92bba397-a997-4d22-a52e-6aa6fab940c1"} 2024-05-14T18:24:25.389Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "bff43288-35b3-4407-b832-8211c9e84937"} 2024-05-14T18:24:31.352Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "52d6ffe5-a13c-4c7a-849b-3bb8340cd6dc"} 2024-05-14T18:24:36.369Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "1846f72f-05d6-40ba-ab6a-33e3543b9f74"} 2024-05-14T18:24:41.749Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "9f49adf9-e3a7-4981-bd9f-c283c57e3e64"} 2024-05-14T18:24:46.351Z INFO Creating or updating backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "dd7b0e49-86f0-49bc-a19f-1af480d13665", "name": "fc020-each-min-gcp-cs", "schedule": "* * * * *"} 2024-05-14T18:24:47.114Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "3a171631-b759-4155-8e3c-ad51fe0d50a8"} 2024-05-14T18:24:52.517Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "111e8664-3596-4555-8f82-115233960543"} 2024-05-14T18:24:57.817Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "b1ce759d-ad9f-476d-b439-b0b5985e98c9"} 2024-05-14T18:25:03.817Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "4040f97e-4dac-4af2-9816-483310603152"} 2024-05-14T18:25:09.169Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "102f485d-7da6-48b8-93dd-42ac7aa6b1d0"} 2024-05-14T18:25:14.939Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "c38e0fca-a4a2-4b55-86e9-237488cb081f"} 2024-05-14T18:25:20.481Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "f71cd9b0-e65b-4cbf-8622-308468fad249"} 2024-05-14T18:25:26.426Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "e187764c-4e8c-47be-813a-df1a779dc815"} 2024-05-14T18:25:31.826Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "86704b7c-29fe-4482-b88e-7261960d03a7"} 2024-05-14T18:25:37.133Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "dbde7253-cbf8-4f68-86e7-4b59a30e1acb"} 2024-05-14T18:25:42.454Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "adcab265-32c1-430a-ac1d-4f833feda8af"} 2024-05-14T18:25:46.040Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "8c02a36f-2e4b-4cdc-909f-6b28c581a5b1", "Namespace": "scheduled-backup-19207", "Name": "xb-cron-scheduled-backup-gcp-cs-2024514182546-q6fav"} 2024-05-14T18:25:48.080Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "eea1ca8e-5cdb-4b4e-9701-edce5236f21e"} 2024-05-14T18:25:49.232Z INFO deleting outdated backup job {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "abb4c96b-feb2-4c10-8788-d69ea3151504", "name": "fc020-each-min-gcp-cs"} 2024-05-14T18:25:51.534Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "abb4c96b-feb2-4c10-8788-d69ea3151504"} 2024-05-14T18:25:58.556Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "2d59bced-45be-49d9-9eab-7aa59a27d78e"} 2024-05-14T18:26:03.890Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "f20e2d28-a287-45ed-8183-e4a5f9b23a5a"} 2024-05-14T18:26:08.975Z ERROR sync users {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "6b8b201d-f226-49a0-8e84-a4c68d20313a", "error": "exec syncusers: command terminated with exit code 137 / \nSyncing user accounts from PXC(scheduled-backup-pxc-0.scheduled-backup-pxc.scheduled-backup-19207.svc.cluster.local:3306) to ProxySQL\n / ", "errorVerbose": "exec syncusers: command terminated with exit code 137 / \nSyncing user accounts from PXC(scheduled-backup-pxc-0.scheduled-backup-pxc.scheduled-backup-19207.svc.cluster.local:3306) to ProxySQL\n / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1246\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1248 2024-05-14T18:26:13.649Z ERROR sync users {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "49c6d493-cb17-4a96-a2e0-1b614384f6cb", "error": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") / / ", "errorVerbose": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") / / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1246\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1248 2024-05-14T18:26:16.230Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "13e6841b-abb2-43ad-8ab2-f2bf257d52c6"} 2024-05-14T18:26:16.231Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "13e6841b-abb2-43ad-8ab2-f2bf257d52c6", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:26:16.236Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "709d9a82-6122-487c-88a9-aba2b2619403"} 2024-05-14T18:26:16.236Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "709d9a82-6122-487c-88a9-aba2b2619403", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:26:16.246Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "69fde282-e7c5-438f-8b3b-da1d8981ee71"} 2024-05-14T18:26:16.247Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "69fde282-e7c5-438f-8b3b-da1d8981ee71", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:26:16.267Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "e0953d24-9b20-4ed0-afd9-e57c8cde3605"} 2024-05-14T18:26:16.267Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "e0953d24-9b20-4ed0-afd9-e57c8cde3605", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:26:16.308Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "9b221c70-c1ed-43c4-bc43-d8563b20e122"} 2024-05-14T18:26:16.308Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "9b221c70-c1ed-43c4-bc43-d8563b20e122", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:26:16.389Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "33fe5714-8070-4b5e-8e64-f91585412b98"} 2024-05-14T18:26:16.389Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "33fe5714-8070-4b5e-8e64-f91585412b98", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:26:16.550Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "805201c1-6872-41cc-bbfa-6f1cab714dbe"} 2024-05-14T18:26:16.550Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "805201c1-6872-41cc-bbfa-6f1cab714dbe", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:26:16.871Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "f4aa35f6-8ca3-497d-9b89-945f1b01c22f"} 2024-05-14T18:26:16.871Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "f4aa35f6-8ca3-497d-9b89-945f1b01c22f", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:26:17.512Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "8a7dcc25-ed90-4ca4-b500-eb6a8c114d37"} 2024-05-14T18:26:17.512Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "8a7dcc25-ed90-4ca4-b500-eb6a8c114d37", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:26:18.793Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "465f5b99-3559-42f8-8f39-72245406ea28"} 2024-05-14T18:26:18.793Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "465f5b99-3559-42f8-8f39-72245406ea28", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:26:20.468Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "2f5c2938-730e-472e-b29b-eee26e94c982", "err": "failed to connect to pod scheduled-backup-pxc-0: dial tcp 10.116.128.4:33062: connect: connection refused"} 2024-05-14T18:26:20.952Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "20aed12d-fd75-483e-aa02-0d8baaaaf9dd", "err": "failed to connect to pod scheduled-backup-pxc-0: dial tcp 10.116.128.4:33062: connect: connection refused"} 2024-05-14T18:26:21.354Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "3d75d42f-23f1-481c-8752-74102823f966"} 2024-05-14T18:26:21.354Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "3d75d42f-23f1-481c-8752-74102823f966", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:26:25.762Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "692d3f2f-d08c-4a11-af61-d660f66042c3", "primary name": "scheduled-backup-pxc-0.scheduled-backup-pxc.scheduled-backup-19207.svc.cluster.local"} 2024-05-14T18:26:25.958Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "3afefbfd-f378-4f6e-9d84-b55909de3dca", "primary name": "scheduled-backup-pxc-0.scheduled-backup-pxc.scheduled-backup-19207.svc.cluster.local"} 2024-05-14T18:26:26.476Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "716f60c4-1940-4c74-ae12-98473bd44d6e"} 2024-05-14T18:26:26.476Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "716f60c4-1940-4c74-ae12-98473bd44d6e", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:26:30.982Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "0fc93bc9-e7f4-4b61-ba5e-e24fda7a1aa2", "primary name": "scheduled-backup-pxc-0.scheduled-backup-pxc.scheduled-backup-19207.svc.cluster.local"} 2024-05-14T18:26:36.145Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "b4e5775d-139f-4609-8f59-8db62f944efb", "primary name": "scheduled-backup-pxc-0.scheduled-backup-pxc.scheduled-backup-19207.svc.cluster.local"} 2024-05-14T18:26:36.716Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "a94297b0-3018-4120-8b78-3919f6e7ef3d"} 2024-05-14T18:26:36.716Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "a94297b0-3018-4120-8b78-3919f6e7ef3d", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 2024-05-14T18:26:51.322Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "scheduled-backup-19207", "name": "scheduled-backup", "reconcileID": "b4874cce-26ca-4383-ab61-affdd27d299c", "primary name": "scheduled-backup-pxc-0.scheduled-backup-pxc.scheduled-backup-19207.svc.cluster.local"} 2024-05-14T18:26:57.197Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "b46dfacf-3935-4993-9ab2-691934de9fec"} 2024-05-14T18:26:57.197Z ERROR Reconciler error {"controller": "pxcbackup-controller", "namespace": "scheduled-backup-19207", "name": "cron-scheduled-backup-gcp-cs-2024514182546-q6fav", "reconcileID": "b46dfacf-3935-4993-9ab2-691934de9fec", "error": "failed to run backup: allowUnsafeConfigurations must be true to run backup on cluster with status initializing", "errorVerbose": "allowUnsafeConfigurations must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1.(*PerconaXtraDBCluster).CanBackup\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1/pxc_types.go:1453\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:162\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to run backup\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:163\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 max retry count 1 reached. something went wrong with operator or kubernetes cluster