Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/logs/restore-to-encrypted-cluster-s3-8-0.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + main + '[' -n '' ']' + create_infra restore-to-encrypted-cluster-s3-17349 + local ns=restore-to-encrypted-cluster-s3-17349 + '[' -n pxc-operator ']' + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + [[ -z restore-to-encrypted-cluster-s3-18612 ]] + [[ -z some-name ]] + kubectl_bin patch pxc -n restore-to-encrypted-cluster-s3-18612 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.4YZP8PiI1C ++ mktemp + local LAST_ERR=/tmp/tmp.BDOFTySGxC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc -n restore-to-encrypted-cluster-s3-18612 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4YZP8PiI1C perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.BDOFTySGxC + rm /tmp/tmp.4YZP8PiI1C /tmp/tmp.BDOFTySGxC + return 0 + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + [[ -z restore-to-encrypted-cluster-s3-18612 ]] + [[ -z on-demand-backup-aws-s3 ]] + kubectl_bin patch pxc-backup -n restore-to-encrypted-cluster-s3-18612 on-demand-backup-aws-s3 --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.sWQHEqY94s ++ mktemp + local LAST_ERR=/tmp/tmp.bOK92nTa6j + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc-backup -n restore-to-encrypted-cluster-s3-18612 on-demand-backup-aws-s3 --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sWQHEqY94s perconaxtradbclusterbackup.pxc.percona.com/on-demand-backup-aws-s3 patched (no change) + cat /tmp/tmp.bOK92nTa6j + rm /tmp/tmp.sWQHEqY94s /tmp/tmp.bOK92nTa6j + return 0 + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found perconaxtradbcluster.pxc.percona.com "some-name" deleted from restore-to-encrypted-cluster-s3-18612 namespace + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found perconaxtradbclusterbackup.pxc.percona.com "on-demand-backup-aws-s3" deleted from restore-to-encrypted-cluster-s3-18612 namespace + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found perconaxtradbclusterrestore.pxc.percona.com "on-demand-backup-aws-s3" deleted from restore-to-encrypted-cluster-s3-18612 namespace + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.i9Z1v31NIl + xargs kubectl delete ns + awk '{print$1}' ++ mktemp + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + local LAST_ERR=/tmp/tmp.XBZTukuADt + local exit_status=0 ++ mktemp + local LAST_OUT=/tmp/tmp.TGZ1OLYae2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator ++ mktemp + local LAST_ERR=/tmp/tmp.3L5Dxtitmv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TGZ1OLYae2 + cat /tmp/tmp.3L5Dxtitmv + rm /tmp/tmp.TGZ1OLYae2 /tmp/tmp.3L5Dxtitmv + return 0 namespace "restore-to-encrypted-cluster-s3-18612" deleted namespace "vault-service-1-22764" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.i9Z1v31NIl namespace "pxc-operator" deleted + cat /tmp/tmp.XBZTukuADt + rm /tmp/tmp.i9Z1v31NIl /tmp/tmp.XBZTukuADt + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.IGukMhGnN2 ++ mktemp + local LAST_ERR=/tmp/tmp.YWddO80KZA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IGukMhGnN2 namespace/pxc-operator created + cat /tmp/tmp.YWddO80KZA + rm /tmp/tmp.IGukMhGnN2 /tmp/tmp.YWddO80KZA + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.349oMIjZa4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.lDWnfaS5up ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.349oMIjZa4 ++ cat /tmp/tmp.lDWnfaS5up ++ rm /tmp/tmp.349oMIjZa4 /tmp/tmp.lDWnfaS5up ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster9 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.zruVJ4SFDC ++ mktemp + local LAST_ERR=/tmp/tmp.ZNdoGJgK3a + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster9 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zruVJ4SFDC Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster9" modified. + cat /tmp/tmp.ZNdoGJgK3a + rm /tmp/tmp.zruVJ4SFDC /tmp/tmp.ZNdoGJgK3a + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.x3HQ9Ql810 ++ mktemp + local LAST_ERR=/tmp/tmp.Y13jtu6wlo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.x3HQ9Ql810 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.Y13jtu6wlo + rm /tmp/tmp.x3HQ9Ql810 /tmp/tmp.Y13jtu6wlo + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.ynaXAAR3Vm ++ mktemp + local LAST_ERR=/tmp/tmp.QC5JKq0Voo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ynaXAAR3Vm customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.QC5JKq0Voo E0702 04:18:03.711836 26496 reflector.go:227] "Failed to watch" err="Get \"https://35.224.82.89/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusters.pxc.percona.com&resourceVersion=1782965881377625000&timeoutSeconds=441&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.ynaXAAR3Vm /tmp/tmp.QC5JKq0Voo + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.yaLNEHnfjq ++ mktemp + local LAST_ERR=/tmp/tmp.BQ9ETYq347 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yaLNEHnfjq customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.BQ9ETYq347 E0702 04:18:06.349678 26905 reflector.go:227] "Failed to watch" err="Get \"https://35.224.82.89/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterbackups.pxc.percona.com&resourceVersion=1782965881377625000&timeoutSeconds=528&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.yaLNEHnfjq /tmp/tmp.BQ9ETYq347 + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.zPpOrv32Rr ++ mktemp + local LAST_ERR=/tmp/tmp.eImGN3WrJl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zPpOrv32Rr customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.eImGN3WrJl E0702 04:18:09.289403 27345 reflector.go:227] "Failed to watch" err="Get \"https://35.224.82.89/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterrestores.pxc.percona.com&resourceVersion=1782965886377650000&timeoutSeconds=300&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.zPpOrv32Rr /tmp/tmp.eImGN3WrJl + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-rbac.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.fWQWt8XJAK ++ mktemp + local LAST_ERR=/tmp/tmp.kR2Ce15kdw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fWQWt8XJAK clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.kR2Ce15kdw + rm /tmp/tmp.fWQWt8XJAK /tmp/tmp.kR2Ce15kdw + return 0 + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-operator.yaml ++ mktemp + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + local LAST_OUT=/tmp/tmp.ZuMWwowRnS ++ mktemp + local LAST_ERR=/tmp/tmp.WJL7HAhrVz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZuMWwowRnS deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.WJL7HAhrVz + rm /tmp/tmp.ZuMWwowRnS /tmp/tmp.WJL7HAhrVz + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.PjjYOcg1ll ++ mktemp + local LAST_ERR=/tmp/tmp.mg5rq5hG0S + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PjjYOcg1ll pod/percona-xtradb-cluster-operator-65b5d85575-dr6cv condition met + cat /tmp/tmp.mg5rq5hG0S E0702 04:18:31.857603 30873 reflector.go:227] "Failed to watch" err="Get \"https://35.224.82.89/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-65b5d85575-dr6cv&resourceVersion=1782965911498975000&timeoutSeconds=592&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.PjjYOcg1ll /tmp/tmp.mg5rq5hG0S + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ mktemp ++ local LAST_OUT=/tmp/tmp.A3m44trI94 +++ mktemp ++ local LAST_ERR=/tmp/tmp.wbozJGStWU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A3m44trI94 ++ cat /tmp/tmp.wbozJGStWU ++ rm /tmp/tmp.A3m44trI94 /tmp/tmp.wbozJGStWU ++ return 0 + wait_pod percona-xtradb-cluster-operator-65b5d85575-dr6cv 480 pxc-operator + local pod=percona-xtradb-cluster-operator-65b5d85575-dr6cv + local max_retry=480 + local ns=pxc-operator ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo percona-xtradb-cluster-operator-65b5d85575-dr6cv + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-65b5d85575-dr6cv condition met E0702 04:18:36.134983 31670 reflector.go:227] "Failed to watch" err="Get \"https://35.224.82.89/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-65b5d85575-dr6cv&resourceVersion=1782965915291273000&timeoutSeconds=302&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-65b5d85575-dr6cv to become Ready.Ok + sleep 3 + create_namespace restore-to-encrypted-cluster-s3-17349 + local namespace=restore-to-encrypted-cluster-s3-17349 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ grep chaos-mesh.org ++ awk '{print $1}' ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces restore-to-encrypted-cluster-s3-17349' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces restore-to-encrypted-cluster-s3-17349 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace restore-to-encrypted-cluster-s3-17349 + awk '{print$1}' ++ mktemp + xargs kubectl delete ns + kubectl_bin get ns + local LAST_OUT=/tmp/tmp.AWmC6UrDyJ + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.7NzSNI8C7l ++ mktemp + local LAST_ERR=/tmp/tmp.flQPAiIfqm + local exit_status=0 ++ mktemp + local LAST_ERR=/tmp/tmp.53dxUAyg6G + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace restore-to-encrypted-cluster-s3-17349 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7NzSNI8C7l + cat /tmp/tmp.53dxUAyg6G + rm /tmp/tmp.7NzSNI8C7l /tmp/tmp.53dxUAyg6G + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace restore-to-encrypted-cluster-s3-17349 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace restore-to-encrypted-cluster-s3-17349 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.AWmC6UrDyJ + cat /tmp/tmp.flQPAiIfqm Error from server (NotFound): namespaces "restore-to-encrypted-cluster-s3-17349" not found + rm /tmp/tmp.AWmC6UrDyJ /tmp/tmp.flQPAiIfqm + return 1 + : + wait_for_delete namespace/restore-to-encrypted-cluster-s3-17349 + local res=namespace/restore-to-encrypted-cluster-s3-17349 + echo -n 'waiting for namespace/restore-to-encrypted-cluster-s3-17349 to be deleted' waiting for namespace/restore-to-encrypted-cluster-s3-17349 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "restore-to-encrypted-cluster-s3-17349" not found + desc 'create namespace restore-to-encrypted-cluster-s3-17349' + set +o xtrace ----------------------------------------------------------------------------------- create namespace restore-to-encrypted-cluster-s3-17349 ----------------------------------------------------------------------------------- + kubectl_bin create namespace restore-to-encrypted-cluster-s3-17349 ++ mktemp + local LAST_OUT=/tmp/tmp.1LgZFKpwft ++ mktemp + local LAST_ERR=/tmp/tmp.ZlzFatY4rK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace restore-to-encrypted-cluster-s3-17349 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1LgZFKpwft namespace/restore-to-encrypted-cluster-s3-17349 created + cat /tmp/tmp.ZlzFatY4rK + rm /tmp/tmp.1LgZFKpwft /tmp/tmp.ZlzFatY4rK + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.cuKMOS9uhJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.LzLzKw18xS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cuKMOS9uhJ ++ cat /tmp/tmp.LzLzKw18xS ++ rm /tmp/tmp.cuKMOS9uhJ /tmp/tmp.LzLzKw18xS ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster9 --namespace=restore-to-encrypted-cluster-s3-17349 ++ mktemp + local LAST_OUT=/tmp/tmp.uuN5ELl4Ff ++ mktemp + local LAST_ERR=/tmp/tmp.RxDK3I4aN5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster9 --namespace=restore-to-encrypted-cluster-s3-17349 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uuN5ELl4Ff Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster9" modified. + cat /tmp/tmp.RxDK3I4aN5 + rm /tmp/tmp.uuN5ELl4Ff /tmp/tmp.RxDK3I4aN5 + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.nwpW6bX1VF ++ mktemp + local LAST_ERR=/tmp/tmp.mvEqnSNyrd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nwpW6bX1VF secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.mvEqnSNyrd + rm /tmp/tmp.nwpW6bX1VF /tmp/tmp.mvEqnSNyrd + return 0 + cluster=some-name + spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.oJnm23o1r5 ++ mktemp + local LAST_ERR=/tmp/tmp.kTzTQ39bKO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oJnm23o1r5 secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.kTzTQ39bKO + rm /tmp/tmp.oJnm23o1r5 /tmp/tmp.kTzTQ39bKO + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.restore-to-encrypted-cluster-s3-17349~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.6suPQYesND ++ mktemp + local LAST_ERR=/tmp/tmp.0khEw0Oqqw + local exit_status=0 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6suPQYesND deployment.apps/pxc-client created + cat /tmp/tmp.0khEw0Oqqw + rm /tmp/tmp.6suPQYesND /tmp/tmp.0khEw0Oqqw + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml + local pvc_name= + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.5M07BRS4wg + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml + /usr/bin/sed -e s~minio-service.#namespace~minio-service.restore-to-encrypted-cluster-s3-17349~ + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + local LAST_ERR=/tmp/tmp.xg9FzPnRD3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5M07BRS4wg perconaxtradbcluster.pxc.percona.com/some-name created + cat /tmp/tmp.xg9FzPnRD3 + rm /tmp/tmp.5M07BRS4wg /tmp/tmp.xg9FzPnRD3 + return 0 + desc 'waiting pods to be running for some-name' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for some-name ----------------------------------------------------------------------------------- ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.f1GGyXcMXi ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7AkimD54HI +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.f1GGyXcMXi +++ cat /tmp/tmp.7AkimD54HI +++ rm /tmp/tmp.f1GGyXcMXi /tmp/tmp.7AkimD54HI +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lltBQ5orOL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GsZSeyYRGp +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.lltBQ5orOL +++ cat /tmp/tmp.GsZSeyYRGp +++ rm /tmp/tmp.lltBQ5orOL /tmp/tmp.GsZSeyYRGp +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo some-name-proxysql ++ return + local proxy=some-name-proxysql ++ get_proxy_size some-name ++ local cluster=some-name ++ local size=0 +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gMfoM8sUCr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uogUmfgHrB +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.gMfoM8sUCr +++ cat /tmp/tmp.uogUmfgHrB +++ rm /tmp/tmp.gMfoM8sUCr /tmp/tmp.uogUmfgHrB +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.oOOHplBaeW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XLnFCWmRaX +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.oOOHplBaeW +++ cat /tmp/tmp.XLnFCWmRaX +++ rm /tmp/tmp.oOOHplBaeW /tmp/tmp.XLnFCWmRaX +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bLgCVKaqaZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.E2jXayReAb +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.bLgCVKaqaZ +++ cat /tmp/tmp.E2jXayReAb +++ rm /tmp/tmp.bLgCVKaqaZ /tmp/tmp.E2jXayReAb +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running some-name-pxc 3 + local name=some-name-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z some-name-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for some-name-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for some-name-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ echo some-name-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met waiting for pod/some-name-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-1 480 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ echo some-name-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-2 480 + local pod=some-name-pxc-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo some-name-pxc-2 + local container=pxc + set +o xtrace pod/some-name-pxc-2 condition met waiting for pod/some-name-pxc-2 to become Ready.Ok + wait_for_running some-name-proxysql 2 + local name=some-name-proxysql + local size=2 + local last_pod + local max_retry=480 + [[ -z some-name-proxysql ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for some-name-proxysql' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for some-name-proxysql ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-proxysql-0 480 + local pod=some-name-proxysql-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo some-name-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace pod/some-name-proxysql-0 condition met E0702 04:23:45.276919 13206 reflector.go:227] "Failed to watch" err="Get \"https://35.224.82.89/api/v1/namespaces/restore-to-encrypted-cluster-s3-17349/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-proxysql-0&resourceVersion=1782966221325291000&timeoutSeconds=401&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/some-name-proxysql-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-proxysql-1 480 + local pod=some-name-proxysql-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo some-name-proxysql-1 + local container=proxysql + set +o xtrace pod/some-name-proxysql-1 condition met E0702 04:23:57.654904 15096 reflector.go:227] "Failed to watch" err="Get \"https://35.224.82.89/api/v1/namespaces/restore-to-encrypted-cluster-s3-17349/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-proxysql-1&resourceVersion=1782966236325383000&timeoutSeconds=549&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/some-name-proxysql-1 to become Ready.Ok + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/some-name to be ready' waiting for pxc/some-name to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AdR5IBpx9L +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z1CJd7oUie ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AdR5IBpx9L ++ cat /tmp/tmp.Z1CJd7oUie ++ rm /tmp/tmp.AdR5IBpx9L /tmp/tmp.Z1CJd7oUie ++ return 0 + state=ready ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.R9pvaQqCCn +++ mktemp ++ local LAST_ERR=/tmp/tmp.jAsUWbzJkW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.R9pvaQqCCn ++ cat /tmp/tmp.jAsUWbzJkW ++ rm /tmp/tmp.R9pvaQqCCn /tmp/tmp.jAsUWbzJkW ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.k4m7aiL6Zz ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.fAWNN61WvS +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.k4m7aiL6Zz +++++ cat /tmp/tmp.fAWNN61WvS +++++ rm /tmp/tmp.k4m7aiL6Zz /tmp/tmp.fAWNN61WvS +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.StbKJvtT19 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.awsPXNrvRU +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.StbKJvtT19 +++++ cat /tmp/tmp.awsPXNrvRU +++++ rm /tmp/tmp.StbKJvtT19 /tmp/tmp.awsPXNrvRU +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oUPM0ZmTu8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.LFtzXcHxgc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oUPM0ZmTu8 ++ cat /tmp/tmp.LFtzXcHxgc ++ rm /tmp/tmp.oUPM0ZmTu8 /tmp/tmp.LFtzXcHxgc ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bza00r18NH +++ mktemp ++ local LAST_ERR=/tmp/tmp.JKMqT0KPUF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bza00r18NH ++ cat /tmp/tmp.JKMqT0KPUF ++ rm /tmp/tmp.bza00r18NH /tmp/tmp.JKMqT0KPUF ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vo7ievW0JH +++ mktemp ++ local LAST_ERR=/tmp/tmp.nyjgfs45hn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vo7ievW0JH ++ cat /tmp/tmp.nyjgfs45hn ++ rm /tmp/tmp.vo7ievW0JH /tmp/tmp.nyjgfs45hn ++ return 0 + client_pod=pxc-client-67fc4995bb-6m8hd + wait_pod pxc-client-67fc4995bb-6m8hd + local pod=pxc-client-67fc4995bb-6m8hd + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-67fc4995bb-6m8hd ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-6m8hd condition met E0702 04:24:50.471353 22859 reflector.go:227] "Failed to watch" err="Get \"https://35.224.82.89/api/v1/namespaces/restore-to-encrypted-cluster-s3-17349/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-6m8hd&resourceVersion=1782966286576375000&timeoutSeconds=545&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-6m8hd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data some-name 100500 no-proxy -P3306 + local cluster=some-name + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password some-name root ++ local cluster=some-name ++ local user=root +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Ybh1uj8g2Q ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jTIGkXTDUu +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Ybh1uj8g2Q +++ cat /tmp/tmp.jTIGkXTDUu +++ rm /tmp/tmp.Ybh1uj8g2Q /tmp/tmp.jTIGkXTDUu +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.i8dA0W6Hjf +++ mktemp ++ local LAST_ERR=/tmp/tmp.HAKNb7RDnz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.i8dA0W6Hjf ++ cat /tmp/tmp.HAKNb7RDnz ++ rm /tmp/tmp.i8dA0W6Hjf /tmp/tmp.HAKNb7RDnz ++ return 0 + root_pass=root_password ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.nwWeuWpHeh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lJ5gZrseyQ +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.nwWeuWpHeh +++ cat /tmp/tmp.lJ5gZrseyQ +++ rm /tmp/tmp.nwWeuWpHeh /tmp/tmp.lJ5gZrseyQ +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QyQ2HZWoVv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7VuSAJ1SD6 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.QyQ2HZWoVv +++ cat /tmp/tmp.7VuSAJ1SD6 +++ rm /tmp/tmp.QyQ2HZWoVv /tmp/tmp.7VuSAJ1SD6 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo some-name-proxysql ++ return + proxy=some-name-proxysql + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h some-name-pxc-0.some-name-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZSZPNxn9gp +++ mktemp ++ local LAST_ERR=/tmp/tmp.kYoVfiYpd8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZSZPNxn9gp ++ cat /tmp/tmp.kYoVfiYpd8 ++ rm /tmp/tmp.ZSZPNxn9gp /tmp/tmp.kYoVfiYpd8 ++ return 0 + client_pod=pxc-client-67fc4995bb-6m8hd + wait_pod pxc-client-67fc4995bb-6m8hd + local pod=pxc-client-67fc4995bb-6m8hd + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-6m8hd ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-6m8hd condition met E0702 04:25:20.469255 27184 reflector.go:227] "Failed to watch" err="Get \"https://35.224.82.89/api/v1/namespaces/restore-to-encrypted-cluster-s3-17349/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-6m8hd&resourceVersion=1782966317124270000&timeoutSeconds=409&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-6m8hd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data some-name select-1 no-proxy -P3306 + local cluster=some-name + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password some-name root ++ local cluster=some-name ++ local user=root +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3K5KLOUj1h ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OTRJKkvQCn +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.3K5KLOUj1h +++ cat /tmp/tmp.OTRJKkvQCn +++ rm /tmp/tmp.3K5KLOUj1h /tmp/tmp.OTRJKkvQCn +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ base64 --decode ++ local LAST_OUT=/tmp/tmp.S3PgaZfbcQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.2k4TbfLqJX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.S3PgaZfbcQ ++ cat /tmp/tmp.2k4TbfLqJX ++ rm /tmp/tmp.S3PgaZfbcQ /tmp/tmp.2k4TbfLqJX ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size some-name ++ local cluster=some-name ++ local size=0 +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.879qsZtgco ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WEvjKg5Wlm +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.879qsZtgco +++ cat /tmp/tmp.WEvjKg5Wlm +++ rm /tmp/tmp.879qsZtgco /tmp/tmp.WEvjKg5Wlm +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/restore-to-encrypted-cluster-s3/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/restore-to-encrypted-cluster-s3/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.am4JScMIyC +++ mktemp ++ local LAST_ERR=/tmp/tmp.jjrbip3CVW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.am4JScMIyC ++ cat /tmp/tmp.jjrbip3CVW ++ rm /tmp/tmp.am4JScMIyC /tmp/tmp.jjrbip3CVW ++ return 0 + client_pod=pxc-client-67fc4995bb-6m8hd + wait_pod pxc-client-67fc4995bb-6m8hd + local pod=pxc-client-67fc4995bb-6m8hd + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-6m8hd ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-6m8hd condition met E0702 04:26:14.137192 3185 reflector.go:227] "Failed to watch" err="Get \"https://35.224.82.89/api/v1/namespaces/restore-to-encrypted-cluster-s3-17349/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-6m8hd&resourceVersion=1782966373129369000&timeoutSeconds=454&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-6m8hd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.mP5YwDYKHX/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.mP5YwDYKHX/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/restore-to-encrypted-cluster-s3/compare/select-1.sql /tmp/tmp.mP5YwDYKHX/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-1.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h some-name-pxc-1.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/restore-to-encrypted-cluster-s3/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/restore-to-encrypted-cluster-s3/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-1.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h some-name-pxc-1.some-name-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.X0wZkAszsg +++ mktemp ++ local LAST_ERR=/tmp/tmp.MrxJ8fhMmJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.X0wZkAszsg ++ cat /tmp/tmp.MrxJ8fhMmJ ++ rm /tmp/tmp.X0wZkAszsg /tmp/tmp.MrxJ8fhMmJ ++ return 0 + client_pod=pxc-client-67fc4995bb-6m8hd + wait_pod pxc-client-67fc4995bb-6m8hd + local pod=pxc-client-67fc4995bb-6m8hd + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-6m8hd ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-6m8hd condition met E0702 04:26:32.929354 5524 reflector.go:227] "Failed to watch" err="Get \"https://35.224.82.89/api/v1/namespaces/restore-to-encrypted-cluster-s3-17349/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-6m8hd&resourceVersion=1782966389628532000&timeoutSeconds=590&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-6m8hd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.mP5YwDYKHX/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.mP5YwDYKHX/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/restore-to-encrypted-cluster-s3/compare/select-1.sql /tmp/tmp.mP5YwDYKHX/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-2.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h some-name-pxc-2.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/restore-to-encrypted-cluster-s3/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/restore-to-encrypted-cluster-s3/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-2.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h some-name-pxc-2.some-name-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9OwgwpEbSW +++ mktemp ++ local LAST_ERR=/tmp/tmp.oPJQ0SCBiY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9OwgwpEbSW ++ cat /tmp/tmp.oPJQ0SCBiY ++ rm /tmp/tmp.9OwgwpEbSW /tmp/tmp.oPJQ0SCBiY ++ return 0 + client_pod=pxc-client-67fc4995bb-6m8hd + wait_pod pxc-client-67fc4995bb-6m8hd + local pod=pxc-client-67fc4995bb-6m8hd + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-6m8hd ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-6m8hd condition met E0702 04:26:49.982004 8128 reflector.go:227] "Failed to watch" err="Get \"https://35.224.82.89/api/v1/namespaces/restore-to-encrypted-cluster-s3-17349/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-6m8hd&resourceVersion=1782966406919486000&timeoutSeconds=528&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-6m8hd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.mP5YwDYKHX/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.mP5YwDYKHX/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/restore-to-encrypted-cluster-s3/compare/select-1.sql /tmp/tmp.mP5YwDYKHX/select-1.sql + is_keyring_plugin_in_use some-name + local cluster=some-name + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + kubectl exec some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + grep -E -o 'early-plugin-load=keyring_\w+.so' + return 1 + keyring_plugin_must_not_be_in_use some-name + local cluster=some-name + log 'checking keyring plugin usage' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:27:03+0000]' checking keyring plugin usage [2026-07-02T04:27:03+0000] checking keyring plugin usage + is_keyring_plugin_in_use some-name + local cluster=some-name + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + grep -E -o 'early-plugin-load=keyring_\w+.so' + kubectl exec some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + return 1 + log 'keyring plugin is not used: OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:27:06+0000]' keyring plugin is not used: OK [2026-07-02T04:27:06+0000] keyring plugin is not used: OK + table_must_not_be_encrypted some-name myApp + local cluster=some-name + local table=myApp + log 'checking table encryption: myApp' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:27:06+0000]' checking table encryption: myApp [2026-07-02T04:27:06+0000] checking table encryption: myApp + is_table_encrypted some-name myApp + local cluster=some-name + local table=myApp + run_mysql 'SELECT CREATE_OPTIONS FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME=\"myApp\";' '-h some-name-proxysql -uroot -proot_password' + local 'command=SELECT CREATE_OPTIONS FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME=\"myApp\";' + local 'uri=-h some-name-proxysql -uroot -proot_password' + grep -E -o 'ENCRYPTION=('\''Y'\''|"Y")' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.szVvTClYqe +++ mktemp ++ local LAST_ERR=/tmp/tmp.6ZTIfgcOQw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.szVvTClYqe ++ cat /tmp/tmp.6ZTIfgcOQw ++ rm /tmp/tmp.szVvTClYqe /tmp/tmp.6ZTIfgcOQw ++ return 0 + client_pod=pxc-client-67fc4995bb-6m8hd + wait_pod pxc-client-67fc4995bb-6m8hd + local pod=pxc-client-67fc4995bb-6m8hd + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-6m8hd ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-6m8hd condition met E0702 04:27:10.376879 11321 reflector.go:227] "Failed to watch" err="Get \"https://35.224.82.89/api/v1/namespaces/restore-to-encrypted-cluster-s3-17349/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-6m8hd&resourceVersion=1782966428188892000&timeoutSeconds=380&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-6m8hd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + log 'myApp is not encrypted: OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:27:20+0000]' myApp is not encrypted: OK [2026-07-02T04:27:20+0000] myApp is not encrypted: OK + vault1=vault-service-1-22347 + prepare_vault vault-service-1-22347 + name=vault-service-1-22347 + protocol=http + local platform=kubernetes + [[ -n '' ]] + create_namespace vault-service-1-22347 skip_clean + local namespace=vault-service-1-22347 + local skip_clean_namespace=skip_clean + [[ 1 == 1 ]] + [[ -z skip_clean ]] + '[' -n '' ']' + desc 'cleaned up old namespaces vault-service-1-22347' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces vault-service-1-22347 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace vault-service-1-22347 ++ mktemp + local LAST_OUT=/tmp/tmp.orPhWXJsIc ++ mktemp + local LAST_ERR=/tmp/tmp.LJVcyUW7jF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace vault-service-1-22347 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace vault-service-1-22347 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace vault-service-1-22347 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.orPhWXJsIc + cat /tmp/tmp.LJVcyUW7jF Error from server (NotFound): namespaces "vault-service-1-22347" not found + rm /tmp/tmp.orPhWXJsIc /tmp/tmp.LJVcyUW7jF + return 1 + : + wait_for_delete namespace/vault-service-1-22347 + local res=namespace/vault-service-1-22347 + echo -n 'waiting for namespace/vault-service-1-22347 to be deleted' waiting for namespace/vault-service-1-22347 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "vault-service-1-22347" not found + desc 'create namespace vault-service-1-22347' + set +o xtrace ----------------------------------------------------------------------------------- create namespace vault-service-1-22347 ----------------------------------------------------------------------------------- + kubectl_bin create namespace vault-service-1-22347 ++ mktemp + local LAST_OUT=/tmp/tmp.8FcaWfjjKf ++ mktemp + local LAST_ERR=/tmp/tmp.uOaWgV2nnd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace vault-service-1-22347 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8FcaWfjjKf namespace/vault-service-1-22347 created + cat /tmp/tmp.uOaWgV2nnd + rm /tmp/tmp.8FcaWfjjKf /tmp/tmp.uOaWgV2nnd + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.YGYDS1o3zR +++ mktemp ++ local LAST_ERR=/tmp/tmp.fIW7rWXloG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YGYDS1o3zR ++ cat /tmp/tmp.fIW7rWXloG ++ rm /tmp/tmp.YGYDS1o3zR /tmp/tmp.fIW7rWXloG ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster9 --namespace=vault-service-1-22347 ++ mktemp + local LAST_OUT=/tmp/tmp.49kGLAPL3Z ++ mktemp + local LAST_ERR=/tmp/tmp.Js4QbLkyhu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster9 --namespace=vault-service-1-22347 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.49kGLAPL3Z Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster9" modified. + cat /tmp/tmp.Js4QbLkyhu + rm /tmp/tmp.49kGLAPL3Z /tmp/tmp.Js4QbLkyhu + return 0 + deploy_helm vault-service-1-22347 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository Update Complete. ⎈Happy Helming!⎈ + helm uninstall vault-service-1-22347 Error: uninstall: Release not loaded: vault-service-1-22347: release: not found + : + desc 'install Vault vault-service-1-22347' + set +o xtrace ----------------------------------------------------------------------------------- install Vault vault-service-1-22347 ----------------------------------------------------------------------------------- + '[' http == https ']' + helm install vault-service-1-22347 hashicorp/vault --disable-openapi-validation --version 0.30.0 --namespace vault-service-1-22347 --set dataStorage.enabled=false --set global.platform=kubernetes NAME: vault-service-1-22347 LAST DEPLOYED: Thu Jul 2 04:27:39 2026 NAMESPACE: vault-service-1-22347 STATUS: deployed REVISION: 1 NOTES: Thank you for installing HashiCorp Vault! Now that you have deployed Vault, you should look over the docs on using Vault with Kubernetes available here: https://developer.hashicorp.com/vault/docs Your release is named vault-service-1-22347. To learn more about the release, try: $ helm status vault-service-1-22347 $ helm get manifest vault-service-1-22347 + [[ -n '' ]] + set +o xtrace pod/vault-service-1-22347-0.....{"running":{"startedAt":"2026-07-02T04:28:00Z"}} + kubectl_bin exec -it vault-service-1-22347-0 -- vault operator init -tls-skip-verify -key-shares=1 -key-threshold=1 -format=json ++ mktemp + local LAST_OUT=/tmp/tmp.cfTBRNOP9j ++ mktemp + local LAST_ERR=/tmp/tmp.pfTFMHwTFN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec -it vault-service-1-22347-0 -- vault operator init -tls-skip-verify -key-shares=1 -key-threshold=1 -format=json + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cfTBRNOP9j + cat /tmp/tmp.pfTFMHwTFN Unable to use a TTY - input is not a terminal or the right kind of file + rm /tmp/tmp.cfTBRNOP9j /tmp/tmp.pfTFMHwTFN + return 0 ++ jq -r '.unseal_keys_b64[]' + unsealKey=ETKdlm3Za8OZpJ7NcKKMIxZQ3jEwp7imJGBixtM27cM= ++ jq -r .root_token + token=hvs.PgRjtvsSlEc1jxJteKizbuBI + sleep 10 + kubectl_bin exec -it vault-service-1-22347-0 -- vault operator unseal -tls-skip-verify ETKdlm3Za8OZpJ7NcKKMIxZQ3jEwp7imJGBixtM27cM= ++ mktemp + local LAST_OUT=/tmp/tmp.5WF95E5hLU ++ mktemp + local LAST_ERR=/tmp/tmp.4t1hH1dmPk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec -it vault-service-1-22347-0 -- vault operator unseal -tls-skip-verify ETKdlm3Za8OZpJ7NcKKMIxZQ3jEwp7imJGBixtM27cM= + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5WF95E5hLU Key Value --- ----- Seal Type shamir Initialized true Sealed false Total Shares 1 Threshold 1 Version 1.19.0 Build Date 2025-03-04T12:36:40Z Storage Type file Cluster Name vault-cluster-0ea4f482 Cluster ID 89d59bd6-4cda-4485-0e97-12a0223afd62 HA Enabled false + cat /tmp/tmp.4t1hH1dmPk Unable to use a TTY - input is not a terminal or the right kind of file + rm /tmp/tmp.5WF95E5hLU /tmp/tmp.4t1hH1dmPk + return 0 + kubectl_bin exec -it vault-service-1-22347-0 -- sh -c 'export VAULT_TOKEN=hvs.PgRjtvsSlEc1jxJteKizbuBI && export VAULT_LOG_LEVEL=trace && vault secrets enable --version=1 -tls-skip-verify -path=secret kv && vault audit enable file file_path=/vault/vault-audit.log' ++ mktemp + local LAST_OUT=/tmp/tmp.QIAssSMKN6 ++ mktemp + local LAST_ERR=/tmp/tmp.0BQrsxTG7s + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec -it vault-service-1-22347-0 -- sh -c 'export VAULT_TOKEN=hvs.PgRjtvsSlEc1jxJteKizbuBI && export VAULT_LOG_LEVEL=trace && vault secrets enable --version=1 -tls-skip-verify -path=secret kv && vault audit enable file file_path=/vault/vault-audit.log' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QIAssSMKN6 Success! Enabled the kv secrets engine at: secret/ Success! Enabled the file audit device at: file/ + cat /tmp/tmp.0BQrsxTG7s Unable to use a TTY - input is not a terminal or the right kind of file + rm /tmp/tmp.QIAssSMKN6 /tmp/tmp.0BQrsxTG7s + return 0 + sleep 10 + local vault_secret=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/vault-secret.yaml + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/vault-secret.yaml + sed -e s/#secret/secret/ + sed -e 's/#vault_url/http:\/\/vault-service-1-22347.vault-service-1-22347.svc.cluster.local:8200/' + sed -e s/#token/hvs.PgRjtvsSlEc1jxJteKizbuBI/ + '[' http == https ']' + /usr/bin/sed -i /#vault_ca/d /tmp/tmp.mP5YwDYKHX/vault-secret.yaml + /usr/bin/sed -i s/#84_vault_ca//g /tmp/tmp.mP5YwDYKHX/vault-secret.yaml + activate_vault vault-service-1-22347 + local name=vault-service-1-22347 + kubectl_bin apply --namespace=restore-to-encrypted-cluster-s3-17349 -f /tmp/tmp.mP5YwDYKHX/vault-secret.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.R63CMY7GAK ++ mktemp + local LAST_ERR=/tmp/tmp.HFimK5lkLW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --namespace=restore-to-encrypted-cluster-s3-17349 -f /tmp/tmp.mP5YwDYKHX/vault-secret.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.R63CMY7GAK secret/some-name-vault created + cat /tmp/tmp.HFimK5lkLW + rm /tmp/tmp.R63CMY7GAK /tmp/tmp.HFimK5lkLW + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.jUeB5dXs1V +++ mktemp ++ local LAST_ERR=/tmp/tmp.HJpabWxc1g ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jUeB5dXs1V ++ cat /tmp/tmp.HJpabWxc1g ++ rm /tmp/tmp.jUeB5dXs1V /tmp/tmp.HJpabWxc1g ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster9 --namespace=restore-to-encrypted-cluster-s3-17349 ++ mktemp + local LAST_OUT=/tmp/tmp.VXL2T8jp64 ++ mktemp + local LAST_ERR=/tmp/tmp.wmm39mJ9at + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster9 --namespace=restore-to-encrypted-cluster-s3-17349 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VXL2T8jp64 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster9" modified. + cat /tmp/tmp.wmm39mJ9at + rm /tmp/tmp.VXL2T8jp64 /tmp/tmp.wmm39mJ9at + return 0 + run_backup some-name on-demand-backup-aws-s3 + local cluster=some-name + local backup=on-demand-backup-aws-s3 + local status=Succeeded + log 'run pxc-backup/on-demand-backup-aws-s3' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:28:32+0000]' run pxc-backup/on-demand-backup-aws-s3 [2026-07-02T04:28:32+0000] run pxc-backup/on-demand-backup-aws-s3 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/restore-to-encrypted-cluster-s3/conf/on-demand-backup-aws-s3.yml ++ mktemp + local LAST_OUT=/tmp/tmp.YiXBSEswEM ++ mktemp + local LAST_ERR=/tmp/tmp.cX4vvfRNm3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/restore-to-encrypted-cluster-s3/conf/on-demand-backup-aws-s3.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YiXBSEswEM perconaxtradbclusterbackup.pxc.percona.com/on-demand-backup-aws-s3 created + cat /tmp/tmp.cX4vvfRNm3 + rm /tmp/tmp.YiXBSEswEM /tmp/tmp.cX4vvfRNm3 + return 0 + wait_backup on-demand-backup-aws-s3 Succeeded + local backup=on-demand-backup-aws-s3 + local status=Succeeded + set +o xtrace waiting for pxc-backup/on-demand-backup-aws-s3 to reach Succeeded state........................................................................................................................................................................................................................................................................................................................................................................2026-07-02T04:31:43.231Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.33.12-gke.1059000"} 2026-07-02T04:31:43.231Z INFO setup Manager starting up {"gitCommit": "dcdf1e8aedba334999b2bd5fa9efcdb1f10e79e0", "gitBranch": "PR-2539-dcdf1e8a", "buildTime": "2026-07-02T01:14:56Z", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} 2026-07-02T04:31:43.232Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-07-02T04:31:43.235Z INFO setup Registering Components. 2026-07-02T04:31:43.366Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-07-02T04:31:43.366Z INFO setup Starting the Cmd. 2026-07-02T04:31:43.366Z INFO controller-runtime.metrics Starting metrics server 2026-07-02T04:31:43.366Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-07-02T04:31:43.366Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-07-02T04:31:43.366Z INFO controller-runtime.webhook Starting webhook server 2026-07-02T04:31:43.366Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-07-02T04:31:43.367Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-07-02T04:31:43.367Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-07-02T04:31:44.167Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-07-02T04:32:00.868Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-07-02T04:32:00.869Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-07-02T04:32:00.869Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-07-02T04:32:00.869Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-07-02T04:32:00.869Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-07-02T04:32:00.869Z DEBUG events percona-xtradb-cluster-operator-65b5d85575-27l7k_85b9f91f-47ba-4c98-bfd6-9a27d9130bd3 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"260d55ac-9de0-423b-a9ef-053e2a924b4a","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1782966720862255009"}, "reason": "LeaderElection"} 2026-07-02T04:32:00.970Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-07-02T04:32:00.970Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-07-02T04:32:01.070Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "84694a4c-c365-4a37-bbf9-efa78fda7db8", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:32:01.070Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-07-02T04:32:01.070Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-07-02T04:32:01.070Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-07-02T04:32:01.070Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-07-02T04:32:01.498Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "1417b7fe-a7e7-49e4-a023-80a57a35fd01", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:32:01.662Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "7a380d08-c183-400d-816d-bb69d6de97bc", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:32:06.071Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "0ce0e7d7-47bb-4037-8b38-1ef2c721cd55", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:32:06.778Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "d75c6e5c-ddfe-4b62-a930-dca93184de3b", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:32:11.072Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "067d43ba-d14c-41e9-afe6-4eccdf2c31cd", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:32:11.898Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "b38d1036-3d24-45e9-b5c6-db7eec0fde86", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:32:16.072Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "19111cc0-b868-472d-aa9e-7fdd5b4395a4", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:32:17.051Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "97bd4e84-ed56-4c2d-abe1-7cc02b5411d7", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:32:21.073Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "f1805449-6050-4d4d-ac52-844135244ce5", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:32:22.169Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "f093f78b-e046-48a8-9ee8-da996ad5971e", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:32:26.074Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "509730e9-b44f-4bfb-98ac-37b09cdbaabb", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:32:27.288Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "3a5b33d9-ab62-4b7b-9782-4111b1917b35", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:32:31.075Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "fc8e9599-947b-45ff-a208-710215a0ddcc", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:32:32.418Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "ccb7845b-209c-4275-b014-aee7f325a616", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:32:36.076Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "04397444-244d-4d6b-871b-ae8686fe5d11", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:32:37.547Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "d30ad655-f835-4f8f-817c-44629967b3bd", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:32:41.076Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "a70d912f-40b2-4e6e-bd3c-f8fdb8aabe35", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:32:42.665Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "09e7954f-de3d-44de-b72b-e83ff4e140b5", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:32:42.876Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "37b7d90d-0dfe-4468-9afa-82774fb60b58", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:32:46.077Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "eddf3778-76fd-4829-8e38-0060e84857c5", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:32:48.004Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "2641b3da-eaeb-4313-a81d-f02c7d2b4f3a", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:32:51.077Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "f4f52033-c128-4e27-8dad-dfa18a0a8d0a", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:32:56.078Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "71d755a4-f421-497c-8f45-4c1815497bbd", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:32:58.410Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "7a06e6c0-9d56-41a3-a394-5939c5b64d2f", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:32:58.645Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "397b27de-1f36-485a-b3cd-91056252b691", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:33:01.079Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "5cfd1ad9-f30f-4863-a70a-294301058220", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:33:06.080Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "32ef2f1d-de08-46e0-aebd-e2db2b71af37", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:33:11.081Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "e51a4273-d44f-4229-ab38-d0fd4f8bb14e", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:33:16.082Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "fcdf02eb-0535-4fd1-bc4f-730dace59bbb", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:33:21.083Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "4493c142-8f9f-4242-b25e-b546c65fb735", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:33:24.638Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "b2661ddf-cd40-4362-ae02-416a036d9339", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:33:26.084Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "d8a3b133-abcd-44d1-9359-9a6aec5173dd", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:33:31.085Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "f0c2ae00-5328-4b5b-aa01-aac01b405551", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:33:34.925Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "17fbe76d-09be-43da-a8f2-32391011402c", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:33:36.085Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "80cfb0ec-be08-440e-90ad-c3cb6b6eaa07", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:33:41.086Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "eb2030bc-01ee-4d64-9e9e-18fd98b7eccb", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:33:46.087Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "8d537e7c-cf34-44c1-94f1-5a730185640e", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:33:50.352Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "5a08fde1-0b49-47ec-adbf-81f937688cb1", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:33:51.089Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "20c74391-0505-42e5-ac80-4eda668db71c", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:33:56.089Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "5f8945d8-1263-4fc2-824b-f427837988ae", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:34:00.697Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "3be71828-5f7a-4a20-be7c-5e80b9a075c9", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:34:01.090Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "99dd67bb-cd7c-4f7c-bf4b-e47bec0374e0", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:34:06.092Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "0e77b1d0-527a-438f-be0e-756c930d05af", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:34:10.983Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "4ee40ba2-d3a9-4fce-a929-cce567016116", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:34:11.093Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "821b1c94-29b8-4efc-b432-0de757754bc0", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:34:16.094Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "fcd11919-082c-4627-866a-40155b9e5328", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:34:21.095Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "bde5d64b-7a1a-414f-a42e-e4cfde385ebc", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:34:26.096Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "48025e04-2bf1-4255-af77-74dddb3b4dbd", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:34:26.758Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "b331f715-35c2-46ca-ac1d-45b4aa1f4f79", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:34:31.097Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "c5c4d6fe-2f2c-4fc7-aec8-a1ddb7cb5625", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:34:36.098Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "9b358448-f806-4aec-b2ff-1a35c5e46c4d", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:34:37.093Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "fba650dd-029e-4431-8eed-6e31779be247", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:34:41.098Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "67f2a524-28d4-44ef-8c62-135d3897b3a1", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:34:46.100Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "edf9d157-c8e1-4247-911c-4deb0537a222", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:34:51.101Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "d142c497-4406-4f5c-8a02-dc9555b365a9", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:34:53.087Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "4955b3e4-5999-45cc-ab3c-f653b1a2558f", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:34:56.101Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "2af8bc32-5bfd-4679-8a1f-e4eeca7fd1dc", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:35:01.102Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "3efbf0e2-8232-4df3-aaf3-11e817a813d6", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:35:03.369Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "f975a3f5-46bc-4b28-830f-a4bc3ae5ae68", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:35:06.103Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "2500d639-9b09-4afe-9656-57db2d88f2f1", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:35:11.103Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "dc3b5cb9-d829-458c-9167-4784e36b4e58", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:35:16.103Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "ee06b9cb-a18f-4065-ba5e-a459fd3f945b", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:35:21.104Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "0d27ab09-d93c-4b0f-ae5b-839dfc913ba1", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:35:24.369Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "dc862f5b-9e06-40c7-b040-5198c1a7481a", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:35:26.105Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "acd3a0a2-7082-49c0-8072-5ff5062e88c6", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:35:31.106Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "1aa9429e-81f7-4427-bf4d-cc4e7e1021cc", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:35:34.676Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "d45fc9ee-030c-4c9e-b290-9a7e2ec8a211", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:35:36.107Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "b6faf699-4432-4767-a2df-cd819d2c62c6", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:35:41.108Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "32ba91d4-18b2-4315-ad40-6acc84e1a948", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:35:44.929Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "dfef88c5-d508-4aed-9cfa-25b46ef120ef", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:35:46.109Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "cb3cef8b-fb91-4cb5-9967-2e088be6db46", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:35:51.110Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "14d252af-a887-4738-b158-c6f71f8fb0b5", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:35:55.190Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "ee5915c2-289b-4a10-9836-36638fcddc26", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:35:56.111Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "404c893b-b648-485d-9663-38549eb43f37", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:36:00.307Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "002936a6-8b20-4eb6-86e4-5f8d2210dbe3", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:36:01.112Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "f93a13f7-2752-4a8e-a4af-b417595583bd", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:36:06.113Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "1e4685eb-e5f7-4cf0-9601-94572fcc502d", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:36:10.744Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "3102b22c-2a89-4459-8860-995dc5167b52", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:36:11.113Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "4d708187-0644-491e-9ad7-7ff16164aee6", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:36:16.114Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "3f84b765-fa6c-44ee-8592-856e70b985fb", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:36:21.114Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "23d284b5-252a-4c27-bd2f-9ec521c6ce18", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:36:26.115Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "eb5d0df8-bd26-45e4-93bf-595fdbd89fc2", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:36:26.332Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "631db2f9-3d9e-48a4-a7db-851aceefefa0", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:36:31.116Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "caa47002-97c3-4e02-9efc-5788ca465bb7", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:36:36.116Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "eec67403-dcd0-4be6-9d71-8f1f0d1a2944", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:36:36.593Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "993e8ecc-8675-4a5a-9ab4-55ff09e59c6b", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:36:41.118Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "847f5059-5c1e-4367-8c38-b5d4d433d132", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:36:46.119Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "441f245b-d4d0-4913-84fa-b85c3cd36aaa", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:36:51.120Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "d9f51839-d90a-446c-a529-d64b6d2ac4bc", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:36:56.120Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "901c6c75-09f6-472d-a82d-0a5e7c803298", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:37:01.122Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "cf7bffc1-ef9a-4bc6-acda-1ce2b7d378aa", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:37:06.123Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "dd4ee03a-3c78-4540-aebf-14d93bc17eea", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:37:07.494Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "951d6b38-b9b8-4189-b066-c71bea0ad01b", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:37:11.123Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "c0169bd4-9e93-4f07-85af-195a79e9fd89", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:37:12.611Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "3eecfc10-d602-4403-980a-3ed4f62fe08c", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:37:16.125Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "61c7fc1c-0149-441f-84cb-806f846fbc64", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:37:21.126Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "8053f444-68cf-4656-9fc8-87604728937f", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:37:26.127Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "0cbd696f-6a48-49a5-b058-0e774d92d70b", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:37:28.214Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "a81c074c-0bdc-43a6-9cbd-bdd2623a81e1", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:37:31.128Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "9269446f-0f1a-42c7-b8f2-c3e82162129a", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:37:36.128Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "c958bc94-9ddd-4335-9b2f-712b3dfdc915", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:37:41.129Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "e787c3f1-3ca4-47c1-a232-0913888674ce", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:37:43.919Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "759ce343-11d7-4b4e-8e5a-2d942a106434", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:37:46.130Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "20feaf34-feb6-40ad-b9ff-665823ffd872", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:37:51.131Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "44885b01-6a37-4bc8-817f-325cb9071fde", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:37:54.272Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "4702d523-1ed6-472f-990b-81374962d574", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:37:56.131Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "56dedf00-533c-4e2a-bf53-159a7554d577", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:37:59.550Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "b967b665-86be-4bd2-b312-ff718d45f45c", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:38:01.132Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "3850b5fa-0d15-44fc-bbb2-8414ce6cb0da", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:38:06.133Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "0be2ac2d-a825-4087-a646-ce68478ee6ac", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:38:11.134Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "b3420c74-ffee-499c-9e06-43a3f5a28a10", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:38:14.977Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "6e236e2e-71c4-4102-8196-16dde1d658e7", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:38:16.134Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "9810fa83-1274-4666-81ee-1f997e5dcae5", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:38:21.134Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "f8051e37-3582-4e7d-82cc-4838f5dc6266", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:38:25.301Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "24976062-dca8-4af5-ac10-fc7823287c8b", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:38:26.135Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "63c4d561-058c-41da-b59d-30546a7549ce", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:38:30.411Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "5aebf7fb-1829-46db-93d8-3c2aa56d39b2", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:38:31.136Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "8e52d88a-02ce-4c80-9883-693a68cf2e0b", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:38:36.136Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "4d06206a-630f-4619-a2bc-128b021879b4", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:38:41.138Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "48b12dad-2352-4d6c-a679-39adf704dd8f", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:38:46.138Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "b4214cb7-97d3-437f-8396-805a7993e224", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:38:50.992Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "9dd245ca-eeb5-42e3-be02-b91a8a3fe4e5", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:38:51.139Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "acb1935e-9b98-47a4-a646-0f9c8773994c", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:38:56.140Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "e9b1b23d-db8b-424a-884d-6059d2fbc92c", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:38:56.146Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "98a0f930-3cf8-437c-9fc0-7fbffb43260b", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:39:01.141Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "94eb2c02-4111-4661-897d-246b02966a35", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:39:06.142Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "8479d369-477c-489c-b866-fa486aa29348", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:39:11.143Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "d790d51d-c40d-45bf-87e7-dee113b6663e", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:39:11.565Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "a2349e0e-bb5d-490c-b847-60b01aa94acb", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:39:16.144Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "a426b383-462d-4bab-91fc-79eee72cf92a", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:39:16.702Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "cc9d941e-48e7-40f7-9f75-75149832d3d3", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:39:21.145Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "b850c7df-09ad-422a-b038-84526eb1aab4", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:39:26.146Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "3600e800-7f5d-46f3-a9fd-a48fde9a8486", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:39:31.147Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "ac81c2ed-a372-4feb-b923-1e9f9b286aea", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:39:36.148Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "8c0079c1-610d-409e-9da1-991e9ff75cbb", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:39:37.223Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "4b78236b-5a1c-463a-8b4c-391d9ccc2ce5", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:39:41.150Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "08f8be95-d50b-4113-b076-20418d778f0e", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:39:46.151Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "c4fcee4a-0bd2-41ca-a218-36d7be1c70a6", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:39:51.151Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "56b12abe-1515-4f92-b916-966a86ba3490", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:39:53.005Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "4ec10ab0-958b-4d48-89be-3b364295df55", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:39:56.152Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "e48dda30-d23d-470d-ab65-622177c91f6f", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:40:01.153Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "632ef9e8-8fe4-4df0-bbce-acd03ce3a07d", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:40:06.154Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "b2b85fa4-ba4c-4ab7-87fc-cbb041cd5ee6", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:40:08.410Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "e4fd27d9-723e-4dc8-8c08-6878750eb396", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:40:11.155Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "513187a1-55fd-45f1-84a2-756f454d07a6", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:40:16.156Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "d85905d8-c2cc-4d0a-8bdf-4529ab34679f", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:40:21.157Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "1870ac90-69b9-41e6-a487-380d9be9f227", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:40:26.158Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "6d022e95-656c-4d49-ba5f-b3a69ca553b6", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:40:31.158Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "bffdbc85-6814-4788-8b42-b28d4d9d945e", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:40:34.103Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "13e1e8c4-a242-4bc2-b93b-fc608414f1e5", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:40:36.159Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "00f7d574-b629-42fa-9dbb-04e58cecffb7", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:40:39.387Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "b0516e94-da98-4729-b867-ab472b68d744", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:40:41.159Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "0321732e-d66d-4973-90d8-9c65ebd73ca0", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:40:46.160Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "b1d1e6c0-4b47-41cd-ac27-53d021629f94", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:40:51.160Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "1b561f49-620e-4161-b243-8e550262e75c", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:40:56.161Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "9ff486d1-4eff-4495-ad65-1be50c71079c", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:41:01.162Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "39e6df5b-7fb0-4389-98fc-cddf104b0bc0", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:41:05.280Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "1f6e0e99-f0d5-48a2-9af8-62e3faac7d4d", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:41:06.163Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "841d71ee-5a34-4965-a006-ccfb2a441bed", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:41:10.391Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "4e782b7d-0ae5-4fa9-b72c-ba98f9a28f77", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:41:11.164Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "561ecc89-d589-4e8f-a30a-ea7ec9031b62", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:41:16.166Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "3bc38bc7-8431-4550-9f99-62d7e6f9e50a", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:41:21.166Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "175a91c3-17a3-4b17-9436-9d526744aaa7", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:41:26.166Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "c97d1c68-e385-43d7-96c8-1d836743d961", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:41:31.021Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "563691a9-a951-4011-aec9-943474f4ac16", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:41:31.167Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "c0d797d9-1e0d-4938-a698-f11b1e4b3a2b", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:41:36.147Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "fc4176d7-b95e-436e-a3a6-73a5356dae12", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:41:36.168Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "ac0d3c77-0109-4640-a8bd-25725a2289e9", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:41:41.168Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "6b8e24ad-9648-4934-8dc7-ec73d7946c40", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:41:41.274Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "b94af987-8b7a-4c04-9761-1a72c8c57fd5", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:41:46.169Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "3293275f-298f-4847-9f72-010e971a408d", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:41:51.169Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "254775a7-21dd-48c5-946d-6a40ec601a2e", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:41:51.582Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "e549b191-8ceb-4ef2-9f06-d18ba1ac3916", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:41:56.171Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "e22709f0-b678-431f-941e-c0977ce28881", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:42:01.172Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "4200f6e2-6d09-415d-a63e-c26d2d93af7b", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:42:06.172Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "4e139ae4-8862-453d-a8a4-7708998bbe93", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:42:11.173Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "842030ba-cb77-443d-879b-f7d01d57e6ba", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:42:16.175Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "80f5a276-1fe8-42e7-9cbe-73aa52027e14", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:42:17.332Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "973e3dde-506c-414b-8fd5-00ed024c947a", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:42:21.175Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "971c1a77-3642-4178-825c-80a74d5fab27", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:42:26.176Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "67ba8def-81ba-4c87-b279-3befe2166ec0", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:42:31.178Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "4e3bbe1f-a9ed-4665-9c29-7d49071db91a", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:42:33.020Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "c3edc406-6251-4449-9618-b6bb3d83125d", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:42:36.179Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "4ce47fa1-cf98-45bf-927e-a237cd298840", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:42:41.180Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "4a6037b7-2bda-4c6a-a9b2-53df06c5364f", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:42:46.181Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "3a9fa309-d0ae-440e-80ab-0e1475eb3378", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:42:51.182Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "f6b543d8-c998-4251-bbc7-532da24bd482", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:42:56.183Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "003d0c92-1a41-4e16-87bf-d381c123b9a3", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:43:01.184Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "9375b15d-ddb7-4843-bf83-3ec35920076f", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:43:04.011Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "beafdeb1-6843-469a-836c-6f038e07664d", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:43:06.185Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "25eb3c6b-2322-4465-9e2a-bfea3bb83347", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:43:11.185Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "68f7965d-b387-4e84-8f19-31c2ed18b389", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:43:14.347Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "f3421efd-d409-4a2a-b908-d0acb71e0391", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:43:16.186Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "f148076f-6239-4c33-bfc9-8ecf8fd54777", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:43:21.188Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "9fd4cd2e-a4cc-447e-bc4b-85bdcf34e1ac", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:43:26.189Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "8a904443-fe7d-4dc1-aba2-eefaeded0148", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:43:29.959Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "6b00181d-919a-406b-9c28-b85ed68853b9", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:43:31.190Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "776d0dac-07ee-496f-a87a-d30732738ea0", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:43:36.191Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "81055433-224b-4394-9154-b4213eff4424", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:43:41.191Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "96be9058-33e3-42be-9741-c56da530a5a9", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:43:46.192Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "1ff57989-b894-40ab-9d95-33e8bd5cfc6e", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:43:50.518Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "2d47f9da-17ff-4276-abb4-823daa703b6d", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:43:51.193Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "422749b6-b74d-48fa-bae2-7d252fe7336c", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:43:56.194Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "1663779c-e55c-448f-b299-bc2a7c401036", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:44:00.783Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "some-name", "reconcileID": "9ceca551-719d-414b-8ba3-d59f4afa7916", "primary name": "some-name-pxc-0.some-name-pxc.restore-to-encrypted-cluster-s3-17349.svc.cluster.local"} 2026-07-02T04:44:01.195Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "1a2944e3-93bc-4987-9b21-5d5f1b078688", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:44:06.196Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "39413c0d-08b3-4197-91aa-415230ee9632", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:44:11.197Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "6dad5816-293f-4d92-8dc9-b4b384fe114a", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} 2026-07-02T04:44:16.197Z INFO Cluster is not ready for backup {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"on-demand-backup-aws-s3","namespace":"restore-to-encrypted-cluster-s3-17349"}, "namespace": "restore-to-encrypted-cluster-s3-17349", "name": "on-demand-backup-aws-s3", "reconcileID": "16a202e2-0bf8-4942-b935-0e4793abc2a4", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"} max retry count 360 reached. something went wrong with operator or kubernetes cluster