Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/logs/sst-retry-limit-8-0.log Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1 + main + local cluster=some-name + local joiner_pod=some-name-pxc-3 + donor_candidates=("${cluster}-pxc-1" "${cluster}-pxc-2") + local donor_candidates + create_infra sst-retry-limit-22076 + local ns=sst-retry-limit-22076 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n sst-retry-limit-4111 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.uIFgYMu9nS ++ mktemp + local LAST_ERR=/tmp/tmp.eYGJTnOsTq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uIFgYMu9nS perconaxtradbcluster.pxc.percona.com "some-name" deleted from sst-retry-limit-4111 namespace + cat /tmp/tmp.eYGJTnOsTq + rm /tmp/tmp.uIFgYMu9nS /tmp/tmp.eYGJTnOsTq + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.PKZq5BoYh9 ++ mktemp + local LAST_ERR=/tmp/tmp.BBi0rWy1Zj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PKZq5BoYh9 No resources found + cat /tmp/tmp.BBi0rWy1Zj + rm /tmp/tmp.PKZq5BoYh9 /tmp/tmp.BBi0rWy1Zj + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.j2pYvwCxva ++ mktemp + local LAST_ERR=/tmp/tmp.cbsATAJ6HQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.j2pYvwCxva No resources found + cat /tmp/tmp.cbsATAJ6HQ + rm /tmp/tmp.j2pYvwCxva /tmp/tmp.cbsATAJ6HQ + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + awk '{print$1}' ++ mktemp + local LAST_OUT=/tmp/tmp.WCotVxR1H0 ++ mktemp + local LAST_OUT=/tmp/tmp.6BBbJ9KXKn ++ mktemp + local LAST_ERR=/tmp/tmp.aIqaja3VyM + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.r9ESyqDzhV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6BBbJ9KXKn + cat /tmp/tmp.r9ESyqDzhV + rm /tmp/tmp.6BBbJ9KXKn /tmp/tmp.r9ESyqDzhV + return 0 namespace "sst-retry-limit-4111" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WCotVxR1H0 namespace "pxc-operator" deleted + cat /tmp/tmp.aIqaja3VyM + rm /tmp/tmp.WCotVxR1H0 /tmp/tmp.aIqaja3VyM + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ePvEf7mJat ++ mktemp + local LAST_ERR=/tmp/tmp.f1FTC6lEt3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ePvEf7mJat namespace/pxc-operator created + cat /tmp/tmp.f1FTC6lEt3 + rm /tmp/tmp.ePvEf7mJat /tmp/tmp.f1FTC6lEt3 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.MErsUZhMeN +++ mktemp ++ local LAST_ERR=/tmp/tmp.7bgGjgjlpY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MErsUZhMeN ++ cat /tmp/tmp.7bgGjgjlpY ++ rm /tmp/tmp.MErsUZhMeN /tmp/tmp.7bgGjgjlpY ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2446-ba0335c4-6-cluster3 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.o70aasdwFA ++ mktemp + local LAST_ERR=/tmp/tmp.bA6CdCHdSw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2446-ba0335c4-6-cluster3 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.o70aasdwFA Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2446-ba0335c4-6-cluster3" modified. + cat /tmp/tmp.bA6CdCHdSw + rm /tmp/tmp.o70aasdwFA /tmp/tmp.bA6CdCHdSw + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.o6mK1gt1uS ++ mktemp + local LAST_ERR=/tmp/tmp.PMa8Z1Rb0v + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.o6mK1gt1uS customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.PMa8Z1Rb0v + rm /tmp/tmp.o6mK1gt1uS /tmp/tmp.PMa8Z1Rb0v + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: pxc-operator^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/deploy/cw-rbac.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.GjclZxOgve ++ mktemp + local LAST_ERR=/tmp/tmp.YRHn49vVZk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GjclZxOgve clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.YRHn49vVZk + rm /tmp/tmp.GjclZxOgve /tmp/tmp.YRHn49vVZk + return 0 + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + kubectl_bin apply -f - ++ mktemp + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2446-ba0335c4^' + local LAST_OUT=/tmp/tmp.rIfFuhYofo + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/deploy/cw-operator.yaml ++ mktemp + local LAST_ERR=/tmp/tmp.ygo7WkDR0M + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rIfFuhYofo deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.ygo7WkDR0M + rm /tmp/tmp.rIfFuhYofo /tmp/tmp.ygo7WkDR0M + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.fIhjWbTafn ++ mktemp + local LAST_ERR=/tmp/tmp.frXhNp921O + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fIhjWbTafn pod/percona-xtradb-cluster-operator-8456794cf7-s6vck condition met + cat /tmp/tmp.frXhNp921O + rm /tmp/tmp.fIhjWbTafn /tmp/tmp.frXhNp921O + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rIGIsV0mgP +++ mktemp ++ local LAST_ERR=/tmp/tmp.giVbYdcnjR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rIGIsV0mgP ++ cat /tmp/tmp.giVbYdcnjR ++ rm /tmp/tmp.rIGIsV0mgP /tmp/tmp.giVbYdcnjR ++ return 0 + wait_pod percona-xtradb-cluster-operator-8456794cf7-s6vck 480 pxc-operator + local pod=percona-xtradb-cluster-operator-8456794cf7-s6vck + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-8456794cf7-s6vck ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-8456794cf7-s6vck condition met waiting for pod/percona-xtradb-cluster-operator-8456794cf7-s6vck to become Ready.Ok + sleep 3 + create_namespace sst-retry-limit-22076 + local namespace=sst-retry-limit-22076 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl api-resources ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces sst-retry-limit-22076' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces sst-retry-limit-22076 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace sst-retry-limit-22076 + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + kubectl_bin get ns + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.TuME7aqrsx ++ mktemp + local LAST_OUT=/tmp/tmp.lNYwl2OUfW ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.A29VQRtFCd + local exit_status=0 + local LAST_ERR=/tmp/tmp.zGFZkf16sl + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace sst-retry-limit-22076 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lNYwl2OUfW + cat /tmp/tmp.zGFZkf16sl + rm /tmp/tmp.lNYwl2OUfW /tmp/tmp.zGFZkf16sl + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace sst-retry-limit-22076 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace sst-retry-limit-22076 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.TuME7aqrsx + cat /tmp/tmp.A29VQRtFCd Error from server (NotFound): namespaces "sst-retry-limit-22076" not found + rm /tmp/tmp.TuME7aqrsx /tmp/tmp.A29VQRtFCd + return 1 + : + wait_for_delete namespace/sst-retry-limit-22076 + local res=namespace/sst-retry-limit-22076 + echo -n 'waiting for namespace/sst-retry-limit-22076 to be deleted' waiting for namespace/sst-retry-limit-22076 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "sst-retry-limit-22076" not found + desc 'create namespace sst-retry-limit-22076' + set +o xtrace ----------------------------------------------------------------------------------- create namespace sst-retry-limit-22076 ----------------------------------------------------------------------------------- + kubectl_bin create namespace sst-retry-limit-22076 ++ mktemp + local LAST_OUT=/tmp/tmp.jZXZJb8HzT ++ mktemp + local LAST_ERR=/tmp/tmp.sjz1sXpDEd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace sst-retry-limit-22076 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jZXZJb8HzT namespace/sst-retry-limit-22076 created + cat /tmp/tmp.sjz1sXpDEd + rm /tmp/tmp.jZXZJb8HzT /tmp/tmp.sjz1sXpDEd + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.d7LUHTuGpE +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z9LvrNHC78 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.d7LUHTuGpE ++ cat /tmp/tmp.Z9LvrNHC78 ++ rm /tmp/tmp.d7LUHTuGpE /tmp/tmp.Z9LvrNHC78 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2446-ba0335c4-6-cluster3 --namespace=sst-retry-limit-22076 ++ mktemp + local LAST_OUT=/tmp/tmp.KJHFrdv9UE ++ mktemp + local LAST_ERR=/tmp/tmp.6MEvdtVZM3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2446-ba0335c4-6-cluster3 --namespace=sst-retry-limit-22076 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KJHFrdv9UE Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2446-ba0335c4-6-cluster3" modified. + cat /tmp/tmp.6MEvdtVZM3 + rm /tmp/tmp.KJHFrdv9UE /tmp/tmp.6MEvdtVZM3 + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xV0Dua9oHB ++ mktemp + local LAST_ERR=/tmp/tmp.E1y7PzvWGW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xV0Dua9oHB secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.E1y7PzvWGW + rm /tmp/tmp.xV0Dua9oHB /tmp/tmp.E1y7PzvWGW + return 0 + desc 'start 3-node cluster with SST retry limit' + set +o xtrace ----------------------------------------------------------------------------------- start 3-node cluster with SST retry limit ----------------------------------------------------------------------------------- + spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/conf/some-name.yml 3 10 /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/secrets.yml /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/client.yml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/conf/some-name.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.qfOyF2N8DK ++ mktemp + local LAST_ERR=/tmp/tmp.guWE1VBf6x + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qfOyF2N8DK secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.guWE1VBf6x + rm /tmp/tmp.qfOyF2N8DK /tmp/tmp.guWE1VBf6x + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2446-ba0335c4#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.sst-retry-limit-22076~ ++ mktemp + local LAST_OUT=/tmp/tmp.Bi5r2WoiSU + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/conf/client.yml ++ mktemp + local LAST_ERR=/tmp/tmp.WaDGCJuAVo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Bi5r2WoiSU deployment.apps/pxc-client created + cat /tmp/tmp.WaDGCJuAVo + rm /tmp/tmp.Bi5r2WoiSU /tmp/tmp.WaDGCJuAVo + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/conf/some-name.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/conf/some-name.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/conf/some-name.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/conf/some-name.yml + local pvc_name= ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_OUT=/tmp/tmp.PKPv1jQVNf + /usr/bin/sed -e s~minio-service.#namespace~minio-service.sst-retry-limit-22076~ + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/conf/some-name.yml + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2446-ba0335c4#' ++ mktemp + local LAST_ERR=/tmp/tmp.tsHdsVRPYD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PKPv1jQVNf perconaxtradbcluster.pxc.percona.com/some-name created + cat /tmp/tmp.tsHdsVRPYD + rm /tmp/tmp.PKPv1jQVNf /tmp/tmp.tsHdsVRPYD + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JBiUOkETin ++++ mktemp +++ local LAST_ERR=/tmp/tmp.s24CYVNn5W +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.JBiUOkETin +++ cat /tmp/tmp.s24CYVNn5W +++ rm /tmp/tmp.JBiUOkETin /tmp/tmp.s24CYVNn5W +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DMhqdP50om ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oIksahC7sT +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.DMhqdP50om +++ cat /tmp/tmp.oIksahC7sT +++ rm /tmp/tmp.DMhqdP50om /tmp/tmp.oIksahC7sT +++ return 0 ++ [[ false == \t\r\u\e ]] ++ echo some-name-pxc + local proxy=some-name-pxc + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-22076 ++ mktemp + local LAST_OUT=/tmp/tmp.99htLaVK8Q ++ mktemp + local LAST_ERR=/tmp/tmp.7hRMH2cVk8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-22076 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-22076 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-22076 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.99htLaVK8Q + cat /tmp/tmp.7hRMH2cVk8 error: no matching resources found + rm /tmp/tmp.99htLaVK8Q /tmp/tmp.7hRMH2cVk8 + return 1 + true + wait_for_running some-name-pxc 1 + local name=some-name-pxc + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo some-name-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met waiting for pod/some-name-pxc-0 to become Ready.Ok + wait_for_running some-name-pxc 3 + local name=some-name-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ echo some-name-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met waiting for pod/some-name-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-1 480 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo some-name-pxc-1 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-2 480 + local pod=some-name-pxc-2 + local max_retry=480 + local ns= ++ echo some-name-pxc-2 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-pxc-2 condition met waiting for pod/some-name-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.7hoOP0g1ZR +++ mktemp ++ local LAST_ERR=/tmp/tmp.Cv0EXLpxev ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7hoOP0g1ZR ++ cat /tmp/tmp.Cv0EXLpxev ++ rm /tmp/tmp.7hoOP0g1ZR /tmp/tmp.Cv0EXLpxev ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gDpd2SYFl2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.WJ8HvsGp3V ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gDpd2SYFl2 ++ cat /tmp/tmp.WJ8HvsGp3V ++ rm /tmp/tmp.gDpd2SYFl2 /tmp/tmp.WJ8HvsGp3V ++ return 0 + client_pod=pxc-client-67fc4995bb-hb4td + wait_pod pxc-client-67fc4995bb-hb4td + local pod=pxc-client-67fc4995bb-hb4td + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-hb4td ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-hb4td condition met waiting for pod/pxc-client-67fc4995bb-hb4td to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iLtlBqt5z7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.1u8XO1rloS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iLtlBqt5z7 ++ cat /tmp/tmp.1u8XO1rloS ++ rm /tmp/tmp.iLtlBqt5z7 /tmp/tmp.1u8XO1rloS ++ return 0 + client_pod=pxc-client-67fc4995bb-hb4td + wait_pod pxc-client-67fc4995bb-hb4td + local pod=pxc-client-67fc4995bb-hb4td + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-hb4td ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-hb4td condition met waiting for pod/pxc-client-67fc4995bb-hb4td to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PPyVoh1clt +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ao3uKUPLaZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PPyVoh1clt ++ cat /tmp/tmp.Ao3uKUPLaZ ++ rm /tmp/tmp.PPyVoh1clt /tmp/tmp.Ao3uKUPLaZ ++ return 0 + client_pod=pxc-client-67fc4995bb-hb4td + wait_pod pxc-client-67fc4995bb-hb4td + local pod=pxc-client-67fc4995bb-hb4td + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-67fc4995bb-hb4td + local container= + set +o xtrace pod/pxc-client-67fc4995bb-hb4td condition met waiting for pod/pxc-client-67fc4995bb-hb4td to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.59IhHEf97j/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.59IhHEf97j/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.59IhHEf97j/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JLzSuPfEL0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.WKpnDFHs4U ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JLzSuPfEL0 ++ cat /tmp/tmp.WKpnDFHs4U ++ rm /tmp/tmp.JLzSuPfEL0 /tmp/tmp.WKpnDFHs4U ++ return 0 + client_pod=pxc-client-67fc4995bb-hb4td + wait_pod pxc-client-67fc4995bb-hb4td + local pod=pxc-client-67fc4995bb-hb4td + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-hb4td ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-hb4td condition met waiting for pod/pxc-client-67fc4995bb-hb4td to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.59IhHEf97j/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.59IhHEf97j/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.59IhHEf97j/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0MVhFcqYYp +++ mktemp ++ local LAST_ERR=/tmp/tmp.Gf43ElFyIl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0MVhFcqYYp ++ cat /tmp/tmp.Gf43ElFyIl ++ rm /tmp/tmp.0MVhFcqYYp /tmp/tmp.Gf43ElFyIl ++ return 0 + client_pod=pxc-client-67fc4995bb-hb4td + wait_pod pxc-client-67fc4995bb-hb4td + local pod=pxc-client-67fc4995bb-hb4td + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-hb4td ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-hb4td condition met waiting for pod/pxc-client-67fc4995bb-hb4td to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.59IhHEf97j/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.59IhHEf97j/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2446/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.59IhHEf97j/select-1.sql + is_keyring_plugin_in_use some-name + local cluster=some-name + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + grep -E -o 'early-plugin-load=keyring_\w+.so' + kubectl exec some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + return 1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.secretsName}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RxcaZAXdKV +++ mktemp ++ local LAST_ERR=/tmp/tmp.muvavy9S1L ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RxcaZAXdKV ++ cat /tmp/tmp.muvavy9S1L ++ rm /tmp/tmp.RxcaZAXdKV /tmp/tmp.muvavy9S1L ++ return 0 + secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Hr2zdvFOKd +++ mktemp ++ local LAST_ERR=/tmp/tmp.JQEaTRkv13 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Hr2zdvFOKd ++ cat /tmp/tmp.JQEaTRkv13 ++ rm /tmp/tmp.Hr2zdvFOKd /tmp/tmp.JQEaTRkv13 ++ return 0 + root_pass=root_password + desc 'scale to 4 nodes to trigger SST' + set +o xtrace ----------------------------------------------------------------------------------- scale to 4 nodes to trigger SST ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type=merge -p '{"spec":{"pxc":{"size":4}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.lgXIXcnNth ++ mktemp + local LAST_ERR=/tmp/tmp.ElNKXMXdFG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc some-name --type=merge -p '{"spec":{"pxc":{"size":4}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lgXIXcnNth perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.ElNKXMXdFG + rm /tmp/tmp.lgXIXcnNth /tmp/tmp.ElNKXMXdFG + return 0 + desc 'verify the joiner pod has SST retry support wired in' + set +o xtrace ----------------------------------------------------------------------------------- verify the joiner pod has SST retry support wired in ----------------------------------------------------------------------------------- + local retry + retry=0 + echo -n 'waiting for pod/some-name-pxc-3 container/pxc to start' waiting for pod/some-name-pxc-3 container/pxc to start+ local joiner_container_started ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dEQHd2rzNz +++ mktemp ++ local LAST_ERR=/tmp/tmp.UpSHsa3h5q ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dEQHd2rzNz ++ cat /tmp/tmp.UpSHsa3h5q ++ rm /tmp/tmp.dEQHd2rzNz /tmp/tmp.UpSHsa3h5q ++ return 0 + joiner_container_started= + [[ '' == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ljRX7KL6D1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.GdIAJQeeCz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ljRX7KL6D1 ++ cat /tmp/tmp.GdIAJQeeCz ++ rm /tmp/tmp.ljRX7KL6D1 /tmp/tmp.GdIAJQeeCz ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rTIY6s1oVi +++ mktemp ++ local LAST_ERR=/tmp/tmp.8g9d18GTKJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rTIY6s1oVi ++ cat /tmp/tmp.8g9d18GTKJ ++ rm /tmp/tmp.rTIY6s1oVi /tmp/tmp.8g9d18GTKJ ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 240 ]] ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.ryO9JS4TsN +++ mktemp ++ local LAST_ERR=/tmp/tmp.PFxUM29VoN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ryO9JS4TsN ++ cat /tmp/tmp.PFxUM29VoN ++ rm /tmp/tmp.ryO9JS4TsN /tmp/tmp.PFxUM29VoN ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 240 ]] ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.YVwhhbLHNs +++ mktemp ++ local LAST_ERR=/tmp/tmp.O8AzzjxKu9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YVwhhbLHNs ++ cat /tmp/tmp.O8AzzjxKu9 ++ rm /tmp/tmp.YVwhhbLHNs /tmp/tmp.O8AzzjxKu9 ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 240 ]] ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.50mgIpKVZE +++ mktemp ++ local LAST_ERR=/tmp/tmp.2fklSar0w9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.50mgIpKVZE ++ cat /tmp/tmp.2fklSar0w9 ++ rm /tmp/tmp.50mgIpKVZE /tmp/tmp.2fklSar0w9 ++ return 0 + joiner_container_started=true + [[ true == \t\r\u\e ]] + echo + desc 'interrupt SST twice so the joiner exhausts allowed retries' + set +o xtrace ----------------------------------------------------------------------------------- interrupt SST twice so the joiner exhausts allowed retries ----------------------------------------------------------------------------------- + interrupt_donor_during_sst root_password some-name-pxc-3 0 some-name-pxc-1 some-name-pxc-2 + local root_pass=root_password + local joiner_pod=some-name-pxc-3 + local current_restart_count=0 + local target_restart_count=1 + local donor_pod + donor_candidates=("${@:4}") + local donor_candidates + local retry=0 + local observed_restart_count + echo -n 'waiting for pod/some-name-pxc-3 restartCount=0 SST attempt' waiting for pod/some-name-pxc-3 restartCount=0 SST attempt+ : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=0 + [[ 0 == \0 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + grep -q 'Waiting for SST streaming to complete!' + kubectl_bin logs some-name-pxc-3 -c pxc + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] + : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=0 + [[ 0 == \0 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + echo + break ++ detect_active_donor root_password 120 some-name-pxc-1 some-name-pxc-2 ++ local root_pass=root_password ++ shift ++ local max_retry=120 ++ shift ++ local retry=0 ++ local donor_pod ++ local donor_rejects_queries ++ local local_state ++ echo -n 'waiting to detect active donor' waiting to detect active donor++ : ++ for donor_pod in '"$@"' +++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace +++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}' ++ donor_rejects_queries=OFF +++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace +++ awk '$1 == "wsrep_local_state_comment" {print $2}' ++ local_state=Donor/Desynced ++ [[ OFF == \O\N ]] ++ [[ Donor/Desynced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]] ++ echo some-name-pxc-1 ++ return 0 + donor_pod=some-name-pxc-1 + echo 'interrupting donor pod/some-name-pxc-1 during SST attempt 1' interrupting donor pod/some-name-pxc-1 during SST attempt 1 + kubectl_bin delete pod some-name-pxc-1 --wait=false --grace-period=0 --force ++ mktemp + local LAST_OUT=/tmp/tmp.ndU9up113V ++ mktemp + local LAST_ERR=/tmp/tmp.5U801txe46 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod some-name-pxc-1 --wait=false --grace-period=0 --force + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ndU9up113V pod "some-name-pxc-1" force deleted from sst-retry-limit-22076 namespace + cat /tmp/tmp.5U801txe46 Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.ndU9up113V /tmp/tmp.5U801txe46 + return 0 + wait_for_pxc_restart_count some-name-pxc-3 1 600 + local pod=some-name-pxc-3 + local expected=1 + local max_retry=600 + local retry=0 + echo -n 'waiting for pod/some-name-pxc-3 restartCount to reach 1' waiting for pod/some-name-pxc-3 restartCount to reach 1++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=7 + [[ 7 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=8 + [[ 8 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=9 + [[ 9 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=10 + [[ 10 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=11 + [[ 11 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=12 + [[ 12 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=13 + [[ 13 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=14 + [[ 14 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=15 + [[ 15 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=16 + [[ 16 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=17 + [[ 17 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=18 + [[ 18 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=19 + [[ 19 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=20 + [[ 20 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=21 + [[ 21 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=22 + [[ 22 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=23 + [[ 23 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=24 + [[ 24 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=25 + [[ 25 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=26 + [[ 26 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=27 + [[ 27 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=28 + [[ 28 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=29 + [[ 29 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=30 + [[ 30 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=31 + [[ 31 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=32 + [[ 32 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=33 + [[ 33 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=34 + [[ 34 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=35 + [[ 35 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=36 + [[ 36 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=37 + [[ 37 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=38 + [[ 38 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 1 ]] + echo + wait_pod some-name-pxc-1 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo some-name-pxc-1 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + interrupt_donor_during_sst root_password some-name-pxc-3 1 some-name-pxc-1 some-name-pxc-2 + local root_pass=root_password + local joiner_pod=some-name-pxc-3 + local current_restart_count=1 + local target_restart_count=2 + local donor_pod + donor_candidates=("${@:4}") + local donor_candidates + local retry=0 + local observed_restart_count + echo -n 'waiting for pod/some-name-pxc-3 restartCount=1 SST attempt' waiting for pod/some-name-pxc-3 restartCount=1 SST attempt+ : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=1 + [[ 1 == \1 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + echo + break ++ detect_active_donor root_password 120 some-name-pxc-1 some-name-pxc-2 ++ local root_pass=root_password ++ shift ++ local max_retry=120 ++ shift ++ local retry=0 ++ local donor_pod ++ local donor_rejects_queries ++ local local_state ++ echo -n 'waiting to detect active donor' waiting to detect active donor++ : ++ for donor_pod in '"$@"' +++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}' +++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace ++ donor_rejects_queries=OFF +++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace +++ awk '$1 == "wsrep_local_state_comment" {print $2}' ++ local_state=Donor/Desynced ++ [[ OFF == \O\N ]] ++ [[ Donor/Desynced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]] ++ echo some-name-pxc-1 ++ return 0 + donor_pod=some-name-pxc-1 + echo 'interrupting donor pod/some-name-pxc-1 during SST attempt 2' interrupting donor pod/some-name-pxc-1 during SST attempt 2 + kubectl_bin delete pod some-name-pxc-1 --wait=false --grace-period=0 --force ++ mktemp + local LAST_OUT=/tmp/tmp.eLGQ0pAbO0 ++ mktemp + local LAST_ERR=/tmp/tmp.goIZg300SH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod some-name-pxc-1 --wait=false --grace-period=0 --force + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eLGQ0pAbO0 pod "some-name-pxc-1" force deleted from sst-retry-limit-22076 namespace + cat /tmp/tmp.goIZg300SH Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.eLGQ0pAbO0 /tmp/tmp.goIZg300SH + return 0 + wait_for_pxc_restart_count some-name-pxc-3 2 600 + local pod=some-name-pxc-3 + local expected=2 + local max_retry=600 + local retry=0 + echo -n 'waiting for pod/some-name-pxc-3 restartCount to reach 2' waiting for pod/some-name-pxc-3 restartCount to reach 2++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=7 + [[ 7 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=8 + [[ 8 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=9 + [[ 9 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=10 + [[ 10 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=11 + [[ 11 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=12 + [[ 12 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=13 + [[ 13 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=14 + [[ 14 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=15 + [[ 15 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=16 + [[ 16 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=17 + [[ 17 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=18 + [[ 18 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=19 + [[ 19 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=20 + [[ 20 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=21 + [[ 21 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=22 + [[ 22 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=23 + [[ 23 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=24 + [[ 24 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=25 + [[ 25 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=26 + [[ 26 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=27 + [[ 27 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=28 + [[ 28 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=29 + [[ 29 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=30 + [[ 30 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=31 + [[ 31 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=32 + [[ 32 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=33 + [[ 33 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=34 + [[ 34 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=35 + [[ 35 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=36 + [[ 36 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=37 + [[ 37 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=38 + [[ 38 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=39 + [[ 39 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=40 + [[ 40 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=41 + [[ 41 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=42 + [[ 42 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=43 + [[ 43 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=44 + [[ 44 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=45 + [[ 45 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=46 + [[ 46 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=47 + [[ 47 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=48 + [[ 48 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=49 + [[ 49 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=50 + [[ 50 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 2 -ge 2 ]] + echo + wait_pod some-name-pxc-1 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo some-name-pxc-1 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + desc 'verify the joiner stops restarting after the retry limit is reached' + set +o xtrace ----------------------------------------------------------------------------------- verify the joiner stops restarting after the retry limit is reached ----------------------------------------------------------------------------------- + retry=0 + echo -n 'waiting for pod/some-name-pxc-3 logs to contain '\''SST retry limit reached.'\''' waiting for pod/some-name-pxc-3 logs to contain 'SST retry limit reached.'+ grep -q 'SST retry limit reached.' + kubectl_bin logs some-name-pxc-3 -c pxc + echo + desc 'verify retries stop after the limit' + set +o xtrace ----------------------------------------------------------------------------------- verify retries stop after the limit ----------------------------------------------------------------------------------- + local joiner_phase ++ kubectl_bin get pod some-name-pxc-3 -o 'jsonpath={.status.phase}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n49CuWtEVB +++ mktemp ++ local LAST_ERR=/tmp/tmp.W1yWFuV9HE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o 'jsonpath={.status.phase}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.n49CuWtEVB ++ cat /tmp/tmp.W1yWFuV9HE ++ rm /tmp/tmp.n49CuWtEVB /tmp/tmp.W1yWFuV9HE ++ return 0 + joiner_phase=Running + local joiner_ready ++ jq -r '.status.conditions[] | select(.type == "Ready") | .status' ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.2Bsfh8b2aX +++ mktemp ++ local LAST_ERR=/tmp/tmp.9DZCASYnzK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2Bsfh8b2aX ++ cat /tmp/tmp.9DZCASYnzK ++ rm /tmp/tmp.2Bsfh8b2aX /tmp/tmp.9DZCASYnzK ++ return 0 + joiner_ready=False + [[ Running != \R\u\n\n\i\n\g ]] + [[ False != \F\a\l\s\e ]] + local restart_count_before ++ get_pxc_restart_count some-name-pxc-3 ++ local pod=some-name-pxc-3 ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[] | select(.name == "pxc") | .restartCount' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nAToPAazyT +++ mktemp ++ local LAST_ERR=/tmp/tmp.w3D2g3eIBU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nAToPAazyT ++ cat /tmp/tmp.w3D2g3eIBU ++ rm /tmp/tmp.nAToPAazyT /tmp/tmp.w3D2g3eIBU ++ return 0 + restart_count_before=2 + sleep_with_log 120 + local d=120 + log 'sleeping for 120 seconds' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-04-30T10:13:55+0000]' sleeping for 120 seconds [2026-04-30T10:13:55+0000] sleeping for 120 seconds + sleep 120 + local restart_count_after ++ get_pxc_restart_count some-name-pxc-3 ++ local pod=some-name-pxc-3 ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.CE8lahE95h +++ mktemp ++ jq -r '.status.containerStatuses[] | select(.name == "pxc") | .restartCount' ++ local LAST_ERR=/tmp/tmp.J2JGRZ7M10 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CE8lahE95h ++ cat /tmp/tmp.J2JGRZ7M10 ++ rm /tmp/tmp.CE8lahE95h /tmp/tmp.J2JGRZ7M10 ++ return 0 + restart_count_after=2 + [[ 2 != \2 ]] + desc 'verify donor candidates remain healthy' + set +o xtrace ----------------------------------------------------------------------------------- verify donor candidates remain healthy ----------------------------------------------------------------------------------- + local donor_result + local donor_pod + for donor_pod in '"${donor_candidates[@]}"' ++ run_mysql_local 'SELECT COUNT(*) FROM myApp.myApp;' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 ++ local 'command=SELECT COUNT(*) FROM myApp.myApp;' ++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' ++ local pod=some-name-pxc-1 ++ local container_name= ++ set +o xtrace + donor_result=1 + [[ 1 != \1 ]] + for donor_pod in '"${donor_candidates[@]}"' ++ run_mysql_local 'SELECT COUNT(*) FROM myApp.myApp;' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-2 ++ local 'command=SELECT COUNT(*) FROM myApp.myApp;' ++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' ++ local pod=some-name-pxc-2 ++ local container_name= ++ set +o xtrace + donor_result=1 + [[ 1 != \1 ]] + destroy sst-retry-limit-22076 + local namespace=sst-retry-limit-22076 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v level=info + grep -v 'the object has been modified' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'get backup status: Job.batch' + sort -u + tee /tmp/tmp.59IhHEf97j/operator.log ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ head -1 ++ local LAST_OUT=/tmp/tmp.SoUfvQWHwa +++ mktemp ++ local LAST_ERR=/tmp/tmp.bwlCtWPS1U ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SoUfvQWHwa ++ cat /tmp/tmp.bwlCtWPS1U ++ rm /tmp/tmp.SoUfvQWHwa /tmp/tmp.bwlCtWPS1U ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-8456794cf7-s6vck ++ mktemp + local LAST_OUT=/tmp/tmp.EHTQG5zEqr ++ mktemp + local LAST_ERR=/tmp/tmp.ZtO2vjDsAW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-8456794cf7-s6vck + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EHTQG5zEqr + cat /tmp/tmp.ZtO2vjDsAW + rm /tmp/tmp.EHTQG5zEqr /tmp/tmp.ZtO2vjDsAW + return 0 2026-04-30T10:01:35.130Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-04-30T10:01:35.130Z INFO setup Manager starting up {"gitCommit": "ba0335c41c0b6da445040c92db81b7687ec6d84d", "gitBranch": "PR-2446-ba0335c4", "buildTime": "2026-04-30T06:42:25Z", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"} 2026-04-30T10:01:35.130Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.32.13-gke.1362000"} 2026-04-30T10:01:35.133Z INFO setup Registering Components. 2026-04-30T10:01:35.821Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-04-30T10:01:35.821Z INFO controller-runtime.metrics Starting metrics server 2026-04-30T10:01:35.821Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-04-30T10:01:35.821Z INFO controller-runtime.webhook Starting webhook server 2026-04-30T10:01:35.821Z INFO setup Starting the Cmd. 2026-04-30T10:01:35.821Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-04-30T10:01:35.822Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-04-30T10:01:35.822Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-04-30T10:01:35.822Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-04-30T10:01:35.922Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-04-30T10:01:35.957Z DEBUG events percona-xtradb-cluster-operator-8456794cf7-s6vck_7a3e8248-2916-4e4a-905a-af3efb5c7043 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"43039be8-5f2d-4be5-8528-d2081bb91fbd","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1777543295950927009"}, "reason": "LeaderElection"} 2026-04-30T10:01:35.957Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-04-30T10:01:35.957Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-04-30T10:01:35.957Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-04-30T10:01:35.957Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-04-30T10:01:35.957Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-04-30T10:01:36.058Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-04-30T10:01:36.058Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-04-30T10:01:36.058Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-04-30T10:01:36.058Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-04-30T10:01:36.058Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-04-30T10:01:36.058Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-04-30T10:02:50.724Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "a8bdc428-94b4-4db4-9dae-848cd4fb7ff6", "version": "1.20.0"} 2026-04-30T10:02:51.261Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "a8bdc428-94b4-4db4-9dae-848cd4fb7ff6", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-30T10:02:51.303Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "a8bdc428-94b4-4db4-9dae-848cd4fb7ff6", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-30T10:02:51.370Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "a8bdc428-94b4-4db4-9dae-848cd4fb7ff6", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-30T10:02:51.409Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "a8bdc428-94b4-4db4-9dae-848cd4fb7ff6", "object": "some-name-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-30T10:02:52.131Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "73a0af9a-5b08-4686-b716-12cf6e4300b8", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-30T10:04:13.023Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "12a0a20a-c641-48f0-892f-f7f6787e22df", "user": "operator"} 2026-04-30T10:04:13.062Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "12a0a20a-c641-48f0-892f-f7f6787e22df", "user": "monitor"} 2026-04-30T10:04:13.109Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "12a0a20a-c641-48f0-892f-f7f6787e22df"} 2026-04-30T10:04:13.143Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "12a0a20a-c641-48f0-892f-f7f6787e22df"} 2026-04-30T10:04:13.173Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "12a0a20a-c641-48f0-892f-f7f6787e22df", "user": "xtrabackup"} 2026-04-30T10:04:13.211Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "12a0a20a-c641-48f0-892f-f7f6787e22df"} 2026-04-30T10:04:13.244Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "12a0a20a-c641-48f0-892f-f7f6787e22df", "user": "replication"} 2026-04-30T10:04:13.246Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "12a0a20a-c641-48f0-892f-f7f6787e22df", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:04:18.334Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "73cc54a5-e694-46c7-9239-4f1539e8a0dd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:04:23.431Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "c276813d-248e-4015-9703-27773a3f78ff", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:04:28.532Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "a5f23bc2-bd3f-417e-a05b-c6979c65211c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:04:33.627Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "3b0f270c-3874-44b6-95d5-68b2fb5763ff", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:04:38.731Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "4fba3a29-7998-4a3a-89bc-8afca37c6ed5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:04:43.819Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "a4b433c6-ed9c-4799-bc16-598bcbb1593c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:04:48.932Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "41f985b5-f8a6-4c88-b4c1-0f4a5275a64e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:04:54.034Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "e6999bf1-3956-4dc1-8b99-7c8a10d74080", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:04:59.251Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "a6251f5a-fae2-4938-8e4a-87fcf6186670", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:05:04.646Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "e4292e6a-02e0-4737-9b53-3460ae7fb634", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:05:09.743Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "8e5a71fe-b36e-4e9a-8763-887eb0c02000", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:05:14.839Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "d255aca9-836c-4666-93e0-9cc62ec8766b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:05:19.940Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "d09c046b-8f6b-4f0c-85fd-34b4c5cae824", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:05:25.046Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "72afbbdf-4c97-4628-a63f-9f20d95fed68", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:05:30.321Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "bb1a7182-bb60-4d74-a881-ed53b10e672c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:05:30.717Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "86e6f453-b717-45fb-8502-07d508fc0896", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:05:35.925Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "383d046d-24b1-4243-b829-b843a9616cc8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:05:41.046Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "98206b40-e43f-45b3-a2d9-9090ceac339d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:05:46.189Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "e36558ec-b526-4a27-8ce3-0faa7ba7a9ee", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:05:51.334Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "60714070-16bd-410d-93e9-d22a0e2d315c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:05:56.467Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "4c56bc2d-ee8d-4fc1-951a-575a5230796e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:06:01.604Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "45deccaa-caff-4857-bd68-50efda71fe20", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:06:06.763Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "487c2d4c-d1f8-4327-9062-b0c897efb4d3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:06:11.893Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "30349e94-4225-40c4-87ca-4fe2aeed92b6", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:06:17.087Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "d74c326c-4a43-4958-a271-586dbf1e2772", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:06:22.252Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "6cc91ae9-db33-404d-b986-5263a359fcfa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:06:27.409Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "c0620f35-0c8e-4c03-9db0-1ca19eadf603", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:06:32.567Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "fa468460-8819-4ff4-bd52-7ea37688ff83", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:06:37.712Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "9fe72203-0cb6-4f2e-bfba-920a2252c818", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:06:42.856Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "66074325-a09e-4958-bad2-d5a00f687363", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:06:47.980Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "268581e6-0fc7-4893-aa8a-e46c00c3221c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:06:53.149Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "935eec94-a70d-46d6-a3f5-db624e188f8d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:06:53.884Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "4733ba48-c13a-4355-b003-4fb1d707b528", "user": "root"} 2026-04-30T10:06:53.930Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "4733ba48-c13a-4355-b003-4fb1d707b528", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:06:53.937Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "4733ba48-c13a-4355-b003-4fb1d707b528", "new version": "8.0.43-34.1"} 2026-04-30T10:06:54.753Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "7ee1a5ca-ee09-41e7-9e05-4e62f8aaf775", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:07:00.425Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "73489df1-f44d-49b2-a4de-4de6b7fcad65", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:07:05.644Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "5717685d-0835-4013-a67b-b1fd2d701e67", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:07:10.852Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "1e1643d3-b505-4623-9c2e-05c5f8ae8e28", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:07:16.070Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "4bc096f3-2b3d-4569-93f5-e0c3fb77abe3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:07:21.302Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "29a8d2a4-4d28-4b11-bff8-7d6af99d8d3f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:07:26.536Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "6769c486-d074-44db-8e10-c1597800e863", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:07:31.748Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "361cabeb-b3d5-4d8e-b4b5-92d8edae8f0b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:07:37.001Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "ec073455-042d-47da-86d6-3c9d551bc5e8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:07:42.205Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "8f58671e-09a2-4dcb-b314-46dcf9b28c12", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:07:47.414Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "d7f489c8-4a0a-4f85-8ecd-4bbcfbdeec03", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:07:52.637Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "42c51385-f369-4272-bfb4-2f5ce36f8a5a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:07:57.884Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "00434797-fcd2-4a6f-bfec-00fa7f385994", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:08:03.103Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "d68a0d6d-62a4-469e-9460-86cee7d46ca7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:08:08.320Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "b607bcf7-1f4b-4854-8950-bf7a82ffeb8a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:08:13.525Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "ee5549ab-cb7f-489a-a132-9bafd1eb744f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:08:18.753Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "2544523e-03a8-4e52-9611-dfb8161e7a93", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:08:23.981Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "7fd4aae7-0810-4312-b6e9-aa06f72380ec", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:08:29.247Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "b78a0b0e-7c6e-4953-aac2-d1e7b2834a09", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:08:30.541Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "21c6ae88-aac8-44df-9f02-e1364bc61296", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-04-30T10:08:30.603Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "21c6ae88-aac8-44df-9f02-e1364bc61296", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-04-30T10:08:30.686Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "21c6ae88-aac8-44df-9f02-e1364bc61296", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:08:31.500Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "7f8abb81-7f62-4f85-bd51-d904b7d323d7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:08:37.201Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "458b7ee2-1fe9-4c7e-97da-a220eeaf2e01", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:08:42.374Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "86a518ab-0e5e-4462-b2ac-4f8e264bfe02", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:08:47.564Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "905bef08-74db-4423-b7c0-c2bdf3010ae8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:08:52.737Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "f71a52b9-060d-40c4-b26c-6477f5ee2a64", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:08:57.908Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "04d379af-8c86-4990-b106-3e428ff010ec", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:09:03.103Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "6cefd79a-8160-4723-af0d-51408037a71f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:09:08.246Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "288a3efe-340c-4f37-8181-b16d2cecff74", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:09:08.371Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "84a5d845-e8d2-4be1-a2c2-693031d975b7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:09:13.488Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "fe5ed776-06d7-4c19-b0ee-8c2f3c960c0b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:09:18.600Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "1003470d-00c4-4ab9-b821-15d4e66677cd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:09:23.700Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "28ead828-bdb7-42fd-9b7c-f18e856fa1ff", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:09:28.795Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "b8badde8-0ef5-4293-8d77-3d9765a21b55", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:09:33.926Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "178f16e6-6bc0-470f-826a-8b50c7f30625", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:09:39.045Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "83d54319-120c-4cb6-959c-8eb4596b10ca", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:09:44.366Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "cd61e60e-3326-4f8a-ba21-f1e08c22056b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:09:49.667Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "41c183ba-cd22-42e4-a4ff-e6a05e4f0d33", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:09:54.777Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "3fae8984-0b76-453e-8988-548269d44ba0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:09:59.912Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "fc0ff6eb-b884-4bfd-a2d6-189d5ffd335a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:10:00.093Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "7b95a407-fc77-4dbd-9c3c-257a55458614", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:10:05.787Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "bd79581c-c843-407e-ac9e-d2f2836a1529", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:10:10.983Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "6b373985-892b-4fb3-afdc-8ae9718b2a55", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:10:16.146Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "f25842f7-dc56-4ba6-b937-005e24a4da7e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:10:21.303Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "f8da77d8-f693-432b-b666-3f1e8d6a7aab", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:10:26.480Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "230540f6-8a06-4eb3-ab0f-d41ea2438a79", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:10:31.669Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "d60e74f2-3415-42e6-a4e3-ecfb9b05db9c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:10:36.946Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "c9ac47a6-992e-4608-8a0f-8bf3642a7f69", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:10:42.119Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "6807c2b0-b3cf-4869-9143-2a45ac047078", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:10:47.308Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "16f62eec-1edd-417d-b3d7-28727b2bb7e8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:10:52.486Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "330b590a-c95b-440f-8874-aa7f6c930921", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:10:57.643Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "0fe32736-6d30-4647-973f-7dfdaa912649", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:11:02.799Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "fd31d988-2e38-4161-ae14-ff0e9c5959e4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:11:07.943Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "c682fcd1-a93a-492d-acd5-8a676bb41f48", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:11:13.108Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "07f81727-efc7-427e-9de5-73769e1393a0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:11:18.271Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "45ed01e5-8185-4552-b15f-c9f4cba312fd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:11:23.453Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "d0e8d779-2a34-4026-a1e5-3f07c8170217", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:11:28.596Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "c21d581d-7a68-44ed-a14c-7a7b90d786d6", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:11:28.748Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "ee5963a0-a124-4ab4-9257-5f201c4dda23", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:11:33.856Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "339e7a02-192b-4093-9426-180dac5feb59", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:11:38.947Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "1866b220-2b2d-4ab4-aab3-ae3154101c65", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:11:44.299Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "520d5a95-6dc0-4831-a31d-8cd7c932c769", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:11:49.609Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "ba4ee22c-150d-4e09-b420-34715fd70049", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:11:54.741Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "1018fce3-f563-4c70-a850-2f52c60ce4c7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:11:59.853Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "c043c142-8f30-4cd0-a036-a313890168cd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:12:04.955Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "0c42e56b-00a0-41da-932c-d745a5f2a35c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:12:10.055Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "cea2a327-637f-436c-ba5d-39a66fe13f18", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:12:15.227Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "2049a97b-ffa8-46ed-85ff-5276a2a23d7c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:12:15.426Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "3e9734df-7190-42d6-ae06-b9df5bd0a4ee", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:12:21.090Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "6fe38326-f543-4652-8461-0e11c7e64be2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:12:26.271Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "281f68a3-4e92-454f-8934-eb7db1742641", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:12:31.451Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "1011a4cf-0b9d-47b7-97d9-89e2978a5ee5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:12:36.623Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "b9ec0f5b-930a-4e33-9b1a-92f883e049df", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:12:41.787Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "1a2be938-73db-412a-bfca-4f7ce3206b54", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:12:46.969Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "709539a8-f55a-46a9-833f-a3b03a4b3a7d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:12:52.144Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "3895fcb1-8294-44e7-8ef1-66b1987a9afa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:12:57.291Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "49b5ced7-88de-4218-8772-af867f7b7f92", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:13:02.445Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "2004d687-d8f8-473c-9730-d2c2f55ba68c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:13:07.612Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "31963dc4-1074-4e42-b616-646d1ffc6bc9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:13:12.784Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "d58aee02-6753-4180-af9a-64bd5de26d09", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:13:17.951Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "fbda8029-8597-4563-b359-c4e051ff4ca8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:13:23.122Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "4dc6753c-13a8-4655-a6ae-091d0c7b0fdc", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:13:28.292Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "52fc6d1c-4ddb-4a5e-9906-2b7bf4172363", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:13:33.482Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "b8ebe363-9959-4f06-924b-23fa64e6a652", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:13:33.661Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "fa4be28d-a123-4e4e-9808-0eec673d868d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:13:39.316Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "6d759da7-017a-493c-9350-de46e63872be", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:13:44.603Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "3dae97a0-ad37-4f78-bd4e-298597fde264", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:13:44.800Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "b4f0d46b-931f-4c24-87cc-e247c8f25a8d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:13:50.401Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "17198577-0383-4a61-a430-70f0b5f8f783", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:13:55.569Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "61c421d9-3866-442f-90d5-811269838708", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:14:00.733Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "e29f0077-ab51-4df6-acdd-fcbe6c701ee5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:14:05.937Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "74994e17-d1d8-4803-897d-a95d5af8f1cb", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:14:11.115Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "462c2524-45fe-4a03-84e1-529f23bd2a10", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:14:16.276Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "42e5cfa5-973a-4219-adb6-5e000facede1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:14:21.451Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "6e45edb9-c7dd-4d58-9c1d-4935e4c1d3fd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:14:26.624Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "0c8ad2fa-2dff-44ad-bddd-29b3177cfcb7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:14:31.803Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "4e755c33-b3e3-44d4-9591-f848a049d0a7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:14:36.959Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "ccb25fc0-8ef9-4728-b1de-b4a3338e0c1f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:14:42.131Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "a06e6861-e654-4f0f-998c-4cd10789741f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:14:47.300Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "11880a9f-6d9d-491e-88a9-0a6db8cd6a67", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:14:52.471Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "0d6aae66-35da-4ff8-bbbb-1f1bfc76e7de", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:14:57.638Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "a8b955c5-3a7c-4a35-9c9c-306d3d8af1ed", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:15:02.802Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "122d9186-88a5-4526-9d91-fe4d1d20eb1e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:15:07.955Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "07a96855-de6d-4be6-9588-edc4c106474a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:15:13.146Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "5a99648f-176a-41d6-910d-892208dc5b5d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:15:18.292Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "cde23a8c-a998-474d-8f57-351122a4f5a6", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:15:23.479Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "e4f0c455-dc63-41d1-9fd0-1d05d0b10faf", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:15:28.629Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "82433206-7d40-432e-b6fc-8c372006774d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:15:33.788Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "db8cdc51-906a-41fd-a086-3c37b7cff751", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:15:38.963Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "8d6bd551-8c48-4b76-87cb-d54dbe767f50", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:15:44.399Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "607be55f-376f-43f0-b24d-8144cfa5fece", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:15:49.747Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "ee456da4-90b0-452c-ba09-2ca1f40bf7d3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:15:54.899Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "46c15962-82a5-440c-a4b3-17a296f74739", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T10:16:00.102Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22076"}, "namespace": "sst-retry-limit-22076", "name": "some-name", "reconcileID": "82f81f5f-4307-44fa-958d-49a3f512b4f0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} -  }, -  { -  }, +  }, -  Annotations: map[string]string{ +  Annotations: map[string]string{ -  APIVersion: "apps/v1", -  APIVersion: "apps/v1", +  AvailableReplicas: 0, -  AvailableReplicas: 3, -  CollisionCount: &0, +  CollisionCount: nil, +  CreationTimestamp: v1.Time{}, -  CreationTimestamp: v1.Time{Time: s"2026-04-30 10:02:51 +0000 UTC"}, +  CurrentReplicas: 0, -  CurrentReplicas: 3, +  CurrentRevision: "", -  CurrentRevision: "some-name-pxc-d9d6689bf", -  DefaultMode: &420, -  DefaultMode: &420, +  DefaultMode: nil, +  DefaultMode: nil, +  DeprecatedServiceAccount: "", -  DeprecatedServiceAccount: "default", +  DNSPolicy: "", -  DNSPolicy: "ClusterFirst", -  FieldsType: "FieldsV1", -  FieldsType: "FieldsV1", -  FieldsV1: s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`..., -  FieldsV1: s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`..., +  Generation: 0, -  Generation: 1, +  ManagedFields: nil, -  ManagedFields: []v1.ManagedFieldsEntry{ -  Manager: "kube-controller-manager", -  Manager: "percona-xtradb-cluster-operator", +  ObservedGeneration: 0, -  ObservedGeneration: 1, -  Operation: "Update", -  Operation: "Update", -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJl"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6NCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJl"..., +  PeriodSeconds: 0, -  PeriodSeconds: 10, +  PersistentVolumeClaimRetentionPolicy: nil, -  PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}", +  Phase: "", -  Phase: "Pending", +  PodManagementPolicy: "", -  PodManagementPolicy: "OrderedReady", +  Protocol: "", -  Protocol: "TCP", +  ReadyReplicas: 0, -  ReadyReplicas: 3, +  Replicas: 0, -  Replicas: 3, -  Replicas: &3, +  Replicas: &4, +  ResourceVersion: "", -  ResourceVersion: "1777543610961631016", +  RestartPolicy: "", -  RestartPolicy: "Always", -  RevisionHistoryLimit: &10, +  RevisionHistoryLimit: nil, +  SchedulerName: "", -  SchedulerName: "default-scheduler", -  Subresource: "status", +  TerminationMessagePath: "", -  TerminationMessagePath: "/dev/termination-log", +  TerminationMessagePolicy: "", -  TerminationMessagePolicy: "File", -  Time: s"2026-04-30 10:02:51 +0000 UTC", -  Time: s"2026-04-30 10:06:50 +0000 UTC", -  TopologySpreadConstraints: nil, +  TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, +  UID: "", -  UID: "c587c727-ad3c-4698-b70a-311f5ec0f422", +  UpdatedReplicas: 0, -  UpdatedReplicas: 3, +  UpdateRevision: "", -  UpdateRevision: "some-name-pxc-d9d6689bf", -  VolumeMode: &"Filesystem", +  VolumeMode: nil,   }    },    },    {    },    },    {    },    },    {    },    ... // 16 identical fields    ... // 16 identical fields    ... // 22 identical fields    ... // 2 identical fields    ... // 3 identical fields    ... // 3 identical fields    ... // 3 identical fields    ... // 4 identical fields    ... // 5 identical fields    ... // 6 identical fields    ... // 7 identical fields    ... // 9 identical fields    AccessModes: nil,    ActiveDeadlineSeconds: nil,    Affinity: nil,    Args: {"mysqld"},    AutomountServiceAccountToken: nil,    AWSElasticBlockStore: nil,    AzureFile: nil,    Capacity: nil,    Conditions: nil,    ConfigMap: &v1.ConfigMapVolumeSource{    ContainerPort: 3306,    ContainerPort: 33060,    ContainerPort: 33062,    ContainerPort: 4444,    ContainerPort: 4567,    ContainerPort: 4568,    Containers: []v1.Container{    DataSource: nil,    DataSourceRef: nil,    DeletionGracePeriodSeconds: nil,    DeletionTimestamp: nil,    EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "some-name-env-vars-pxc"}, Optional: &true}}},    Env: {{Name: "PXC_SERVICE", Value: "some-name-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name"}, Key: "xtrabackup"}}}, ...},    EphemeralContainers: nil,    FailureThreshold: 3,    FC: nil,    Finalizers: nil,    GitRepo: nil,    HostAliases: nil,    HostIP: "",    HostPort: 0,    ImagePullPolicy: "IfNotPresent",    InitContainers: []v1.Container{    InitialDelaySeconds: 300,    ISCSI: nil,    Items: nil,    Items: nil,    Labels: nil,    Lifecycle: nil,    LivenessProbe: &v1.Probe{    LocalObjectReference: {Name: "auto-some-name-pxc"},    LocalObjectReference: {Name: "some-name-pxc"},    MinReadySeconds: 0,    Name: "auto-config",    {Name: "bin", VolumeSource: {EmptyDir: &{}}},    Name: "config",    Name: "ist",    Name: "mysql",    Name: "mysql-admin",    Name: "mysql-init-file",    Name: "mysql-users-secret-file",    Name: "mysqlx",    Namespace: "sst-retry-limit-22076",    Name: "ssl",    Name: "ssl-internal",    Name: "sst",    {Name: "tmp", VolumeSource: {EmptyDir: &{}}},    Name: "vault-keyring-secret",    Name: "write-set",    NFS: nil,    NodeName: "",    NodeSelector: nil,    ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "cfd6a52398268173b51d7cdb1331c09a", "percona.com/ssl-internal-hash": "751e59a1c1d94ac67f9d6bb6e756acdc"}},    ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    ObjectMeta: v1.ObjectMeta{    Optional: &false,    Optional: &true,    Optional: &true,    Ordinals: nil,    OS: nil,    Overhead: nil,    OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "some-name", UID: "88511bea-747f-4006-81d2-1de23be1095d", ...}},    Ports: []v1.ContainerPort{    PreemptionPolicy: nil,    ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}},    Quobyte: nil,    ReadinessProbe: &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...},    SecretName: "internal-some-name",    SecretName: "some-name-mysql-init",    SecretName: "some-name-ssl",    SecretName: "some-name-ssl-internal",    SecretName: "some-name-vault",    Secret: &v1.SecretVolumeSource{    SecurityContext: nil,    Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    SelfLink: "",    ServiceAccountName: "default",    ServiceName: "some-name-pxc",    SetHostnameAsFQDN: nil,    Spec: v1.PersistentVolumeClaimSpec{    Spec: v1.PodSpec{    Spec: v1.StatefulSetSpec{    StartupProbe: nil,    Status: v1.PersistentVolumeClaimStatus{    Status: v1.StatefulSetStatus{    StorageClassName: nil,    Subdomain: "",    SuccessThreshold: 1,    Template: v1.PodTemplateSpec{    TerminationGracePeriodSeconds: &600,    TerminationGracePeriodSeconds: nil,    TimeoutSeconds: 5,    Tolerations: nil,    TypeMeta: {},    TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"},    UpdateStrategy: {Type: "RollingUpdate", RollingUpdate: &{Partition: &0}},   &v1.StatefulSet{    VolumeAttributesClassName: nil,    VolumeClaimTemplates: []v1.PersistentVolumeClaim{    VolumeDevices: nil,    VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...},    VolumeName: "",    VolumeSource: v1.VolumeSource{    Volumes: []v1.Volume{    VsphereVolume: nil,    WorkingDir: "", + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n sst-retry-limit-22076 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.0y6vrTIdOs ++ mktemp + local LAST_ERR=/tmp/tmp.ObA7GH87ys + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0y6vrTIdOs perconaxtradbcluster.pxc.percona.com "some-name" deleted from sst-retry-limit-22076 namespace + cat /tmp/tmp.ObA7GH87ys + rm /tmp/tmp.0y6vrTIdOs /tmp/tmp.ObA7GH87ys + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.FjYF8TB6h9 ++ mktemp + local LAST_ERR=/tmp/tmp.IYFteCFDsQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FjYF8TB6h9 No resources found + cat /tmp/tmp.IYFteCFDsQ + rm /tmp/tmp.FjYF8TB6h9 /tmp/tmp.IYFteCFDsQ + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.OWsCnXnreJ ++ mktemp + local LAST_ERR=/tmp/tmp.GlRRxevOAt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OWsCnXnreJ No resources found + cat /tmp/tmp.GlRRxevOAt + rm /tmp/tmp.OWsCnXnreJ /tmp/tmp.GlRRxevOAt + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.zX8fUVOnWe ++ mktemp + local LAST_ERR=/tmp/tmp.mNaWjL7HM6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zX8fUVOnWe validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.mNaWjL7HM6 + rm /tmp/tmp.zX8fUVOnWe /tmp/tmp.mNaWjL7HM6 + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace sst-retry-limit-22076 + rm -rf /tmp/tmp.59IhHEf97j + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.6NB0eULpCi + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- ++ mktemp + local LAST_OUT=/tmp/tmp.lBOKVY3eeN + local LAST_ERR=/tmp/tmp.AMrPRUUbSu + local exit_status=0 ++ seq 0 2 ++ mktemp + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace sst-retry-limit-22076 + local LAST_ERR=/tmp/tmp.rbq4ZwEkCP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator