Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/logs/sst-retry-limit-8-4.log Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1 + main + local cluster=some-name + local joiner_pod=some-name-pxc-3 + donor_candidates=("${cluster}-pxc-1" "${cluster}-pxc-2") + local donor_candidates + create_infra sst-retry-limit-11163 + local ns=sst-retry-limit-11163 + '[' -n pxc-operator ']' + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + kubectl patch pxc -n sst-retry-limit-1147 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.MSMLwnzxXp ++ mktemp + local LAST_ERR=/tmp/tmp.6s1tyABIRP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MSMLwnzxXp perconaxtradbcluster.pxc.percona.com "some-name" deleted from sst-retry-limit-1147 namespace + cat /tmp/tmp.6s1tyABIRP + rm /tmp/tmp.MSMLwnzxXp /tmp/tmp.6s1tyABIRP + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.hUaVvr6u1m ++ mktemp + local LAST_ERR=/tmp/tmp.MdyKHVfjWC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hUaVvr6u1m No resources found + cat /tmp/tmp.MdyKHVfjWC + rm /tmp/tmp.hUaVvr6u1m /tmp/tmp.MdyKHVfjWC + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.Br7Mplt5k0 ++ mktemp + local LAST_ERR=/tmp/tmp.uURGDQISIW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Br7Mplt5k0 No resources found + cat /tmp/tmp.uURGDQISIW + rm /tmp/tmp.Br7Mplt5k0 /tmp/tmp.uURGDQISIW + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl api-resources ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrole ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.k8b1sBIUkQ + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.tzNyhi0gl8 ++ mktemp + local LAST_ERR=/tmp/tmp.C5vEoaNS85 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.ezBGZoybjo + local exit_status=0 + xargs kubectl delete ns + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tzNyhi0gl8 + cat /tmp/tmp.C5vEoaNS85 + rm /tmp/tmp.tzNyhi0gl8 /tmp/tmp.C5vEoaNS85 + return 0 namespace "sst-retry-limit-1147" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.k8b1sBIUkQ namespace "pxc-operator" deleted + cat /tmp/tmp.ezBGZoybjo + rm /tmp/tmp.k8b1sBIUkQ /tmp/tmp.ezBGZoybjo + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.CWLaVCtctD ++ mktemp + local LAST_ERR=/tmp/tmp.CxD4GAekdP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CWLaVCtctD namespace/pxc-operator created + cat /tmp/tmp.CxD4GAekdP + rm /tmp/tmp.CWLaVCtctD /tmp/tmp.CxD4GAekdP + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.qA6GLQStVn +++ mktemp ++ local LAST_ERR=/tmp/tmp.gPESnWUJw8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qA6GLQStVn ++ cat /tmp/tmp.gPESnWUJw8 ++ rm /tmp/tmp.qA6GLQStVn /tmp/tmp.gPESnWUJw8 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-9bec550e-7-cluster12 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.GmMz9MWg5P ++ mktemp + local LAST_ERR=/tmp/tmp.CL7LzwgC3p + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-9bec550e-7-cluster12 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GmMz9MWg5P Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-9bec550e-7-cluster12" modified. + cat /tmp/tmp.CL7LzwgC3p + rm /tmp/tmp.GmMz9MWg5P /tmp/tmp.CL7LzwgC3p + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.mKpGGx2Q8B ++ mktemp + local LAST_ERR=/tmp/tmp.Uual1OPWEy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mKpGGx2Q8B customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.Uual1OPWEy + rm /tmp/tmp.mKpGGx2Q8B /tmp/tmp.Uual1OPWEy + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/deploy/cw-rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.FNeXOoprfM ++ mktemp + local LAST_ERR=/tmp/tmp.YvMxqj0d0O + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FNeXOoprfM clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.YvMxqj0d0O + rm /tmp/tmp.FNeXOoprfM /tmp/tmp.YvMxqj0d0O + return 0 + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/deploy/cw-operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.hwVzezngcE ++ mktemp + local LAST_ERR=/tmp/tmp.5FyhBWX7pP + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2433-9bec550e^' + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hwVzezngcE deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.5FyhBWX7pP + rm /tmp/tmp.hwVzezngcE /tmp/tmp.5FyhBWX7pP + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.gR5WriOBiW ++ mktemp + local LAST_ERR=/tmp/tmp.J9LMCcMpMc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gR5WriOBiW pod/percona-xtradb-cluster-operator-7cd7bb76f4-kzxqr condition met + cat /tmp/tmp.J9LMCcMpMc + rm /tmp/tmp.gR5WriOBiW /tmp/tmp.J9LMCcMpMc + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FtlHNHuZG2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.48M32fjMOh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FtlHNHuZG2 ++ cat /tmp/tmp.48M32fjMOh ++ rm /tmp/tmp.FtlHNHuZG2 /tmp/tmp.48M32fjMOh ++ return 0 + wait_pod percona-xtradb-cluster-operator-7cd7bb76f4-kzxqr 480 pxc-operator + local pod=percona-xtradb-cluster-operator-7cd7bb76f4-kzxqr + local max_retry=480 + local ns=pxc-operator ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo percona-xtradb-cluster-operator-7cd7bb76f4-kzxqr + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-7cd7bb76f4-kzxqr condition met waiting for pod/percona-xtradb-cluster-operator-7cd7bb76f4-kzxqr to become Ready.Ok + sleep 3 + create_namespace sst-retry-limit-11163 + local namespace=sst-retry-limit-11163 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces sst-retry-limit-11163' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces sst-retry-limit-11163 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace sst-retry-limit-11163 ++ mktemp + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + local LAST_OUT=/tmp/tmp.BvbpFZaFNl + xargs kubectl delete ns + kubectl_bin get ns + awk '{print$1}' ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.xltQb1eKVj + local exit_status=0 + local LAST_OUT=/tmp/tmp.6QQIi0J9Ym ++ mktemp + local LAST_ERR=/tmp/tmp.nanqhf1R6R + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace sst-retry-limit-11163 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace sst-retry-limit-11163 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6QQIi0J9Ym + cat /tmp/tmp.nanqhf1R6R + rm /tmp/tmp.6QQIi0J9Ym /tmp/tmp.nanqhf1R6R + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace sst-retry-limit-11163 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.BvbpFZaFNl + cat /tmp/tmp.xltQb1eKVj Error from server (NotFound): namespaces "sst-retry-limit-11163" not found + rm /tmp/tmp.BvbpFZaFNl /tmp/tmp.xltQb1eKVj + return 1 + : + wait_for_delete namespace/sst-retry-limit-11163 + local res=namespace/sst-retry-limit-11163 + echo -n 'waiting for namespace/sst-retry-limit-11163 to be deleted' waiting for namespace/sst-retry-limit-11163 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "sst-retry-limit-11163" not found + desc 'create namespace sst-retry-limit-11163' + set +o xtrace ----------------------------------------------------------------------------------- create namespace sst-retry-limit-11163 ----------------------------------------------------------------------------------- + kubectl_bin create namespace sst-retry-limit-11163 ++ mktemp + local LAST_OUT=/tmp/tmp.DPqZdLuave ++ mktemp + local LAST_ERR=/tmp/tmp.IhkaAK3LjC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace sst-retry-limit-11163 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DPqZdLuave namespace/sst-retry-limit-11163 created + cat /tmp/tmp.IhkaAK3LjC + rm /tmp/tmp.DPqZdLuave /tmp/tmp.IhkaAK3LjC + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.yZ3pqSHDGr +++ mktemp ++ local LAST_ERR=/tmp/tmp.JRzKUtGXe4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yZ3pqSHDGr ++ cat /tmp/tmp.JRzKUtGXe4 ++ rm /tmp/tmp.yZ3pqSHDGr /tmp/tmp.JRzKUtGXe4 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-9bec550e-7-cluster12 --namespace=sst-retry-limit-11163 ++ mktemp + local LAST_OUT=/tmp/tmp.C8YfFVRqnb ++ mktemp + local LAST_ERR=/tmp/tmp.KRuUhSEV2G + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-9bec550e-7-cluster12 --namespace=sst-retry-limit-11163 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.C8YfFVRqnb Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-9bec550e-7-cluster12" modified. + cat /tmp/tmp.KRuUhSEV2G + rm /tmp/tmp.C8YfFVRqnb /tmp/tmp.KRuUhSEV2G + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.GESsUeoNQR ++ mktemp + local LAST_ERR=/tmp/tmp.yZaKwaZKMv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GESsUeoNQR secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.yZaKwaZKMv + rm /tmp/tmp.GESsUeoNQR /tmp/tmp.yZaKwaZKMv + return 0 + desc 'start 3-node cluster with SST retry limit' + set +o xtrace ----------------------------------------------------------------------------------- start 3-node cluster with SST retry limit ----------------------------------------------------------------------------------- + spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/conf/some-name.yml 3 10 /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/secrets.yml /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/conf/some-name.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.PDn4l1hwVb ++ mktemp + local LAST_ERR=/tmp/tmp.4VsNGyeM5g + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PDn4l1hwVb secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.4VsNGyeM5g + rm /tmp/tmp.PDn4l1hwVb /tmp/tmp.4VsNGyeM5g + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml + local pvc_name= + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.sst-retry-limit-11163~ + local LAST_OUT=/tmp/tmp.uMZimi0kSo + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-9bec550e#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' ++ mktemp + local LAST_ERR=/tmp/tmp.G2U0DL6MjM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uMZimi0kSo deployment.apps/pxc-client created + cat /tmp/tmp.G2U0DL6MjM + rm /tmp/tmp.uMZimi0kSo /tmp/tmp.G2U0DL6MjM + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/conf/some-name.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/conf/some-name.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/conf/some-name.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/conf/some-name.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-9bec550e#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/conf/some-name.yml + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.CO3LmfcQDL + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.sst-retry-limit-11163~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.vqJKrxlpAc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CO3LmfcQDL perconaxtradbcluster.pxc.percona.com/some-name created + cat /tmp/tmp.vqJKrxlpAc + rm /tmp/tmp.CO3LmfcQDL /tmp/tmp.vqJKrxlpAc + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.h9YoOm01JF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3wgUem0tm7 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.h9YoOm01JF +++ cat /tmp/tmp.3wgUem0tm7 +++ rm /tmp/tmp.h9YoOm01JF /tmp/tmp.3wgUem0tm7 +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qKibwEOVGT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5JPJWz8QlK +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.qKibwEOVGT +++ cat /tmp/tmp.5JPJWz8QlK +++ rm /tmp/tmp.qKibwEOVGT /tmp/tmp.5JPJWz8QlK +++ return 0 ++ [[ false == \t\r\u\e ]] ++ echo some-name-pxc + local proxy=some-name-pxc + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-11163 ++ mktemp + local LAST_OUT=/tmp/tmp.OgfgLwiQPG ++ mktemp + local LAST_ERR=/tmp/tmp.uDHnnQWxFC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-11163 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-11163 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-11163 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.OgfgLwiQPG + cat /tmp/tmp.uDHnnQWxFC error: no matching resources found + rm /tmp/tmp.OgfgLwiQPG /tmp/tmp.uDHnnQWxFC + return 1 + true + wait_for_running some-name-pxc 1 + local name=some-name-pxc + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ echo some-name-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met waiting for pod/some-name-pxc-0 to become Ready.Ok + wait_for_running some-name-pxc 3 + local name=some-name-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ echo some-name-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met waiting for pod/some-name-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-1 480 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ echo some-name-pxc-1 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-2 480 + local pod=some-name-pxc-2 + local max_retry=480 + local ns= ++ echo some-name-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-2 condition met waiting for pod/some-name-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.F4xPAkizbO +++ mktemp ++ local LAST_ERR=/tmp/tmp.PkPPklxsSI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.F4xPAkizbO ++ cat /tmp/tmp.PkPPklxsSI ++ rm /tmp/tmp.F4xPAkizbO /tmp/tmp.PkPPklxsSI ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6N7mMMGxs1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.wITkShbyht ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6N7mMMGxs1 ++ cat /tmp/tmp.wITkShbyht ++ rm /tmp/tmp.6N7mMMGxs1 /tmp/tmp.wITkShbyht ++ return 0 + client_pod=pxc-client-56fd5498cd-7h2wc + wait_pod pxc-client-56fd5498cd-7h2wc + local pod=pxc-client-56fd5498cd-7h2wc + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-7h2wc ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-7h2wc condition met waiting for pod/pxc-client-56fd5498cd-7h2wc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.65H4nqJkSY +++ mktemp ++ local LAST_ERR=/tmp/tmp.IIu7auPAfZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.65H4nqJkSY ++ cat /tmp/tmp.IIu7auPAfZ ++ rm /tmp/tmp.65H4nqJkSY /tmp/tmp.IIu7auPAfZ ++ return 0 + client_pod=pxc-client-56fd5498cd-7h2wc + wait_pod pxc-client-56fd5498cd-7h2wc + local pod=pxc-client-56fd5498cd-7h2wc + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-7h2wc ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-7h2wc condition met waiting for pod/pxc-client-56fd5498cd-7h2wc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/compare/select-1-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.m3wjO60c3G +++ mktemp ++ local LAST_ERR=/tmp/tmp.U3m17DppWD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.m3wjO60c3G ++ cat /tmp/tmp.U3m17DppWD ++ rm /tmp/tmp.m3wjO60c3G /tmp/tmp.U3m17DppWD ++ return 0 + client_pod=pxc-client-56fd5498cd-7h2wc + wait_pod pxc-client-56fd5498cd-7h2wc + local pod=pxc-client-56fd5498cd-7h2wc + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-56fd5498cd-7h2wc ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-7h2wc condition met waiting for pod/pxc-client-56fd5498cd-7h2wc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.McPJo1bZq6/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.McPJo1bZq6/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.McPJo1bZq6/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/compare/select-1-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CH9hMKIjaD +++ mktemp ++ local LAST_ERR=/tmp/tmp.9ooKuih1uO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CH9hMKIjaD ++ cat /tmp/tmp.9ooKuih1uO ++ rm /tmp/tmp.CH9hMKIjaD /tmp/tmp.9ooKuih1uO ++ return 0 + client_pod=pxc-client-56fd5498cd-7h2wc + wait_pod pxc-client-56fd5498cd-7h2wc + local pod=pxc-client-56fd5498cd-7h2wc + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-7h2wc ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-7h2wc condition met waiting for pod/pxc-client-56fd5498cd-7h2wc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.McPJo1bZq6/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.McPJo1bZq6/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.McPJo1bZq6/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/compare/select-1-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cEYFEhCL5S +++ mktemp ++ local LAST_ERR=/tmp/tmp.LuISGKwzjA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cEYFEhCL5S ++ cat /tmp/tmp.LuISGKwzjA ++ rm /tmp/tmp.cEYFEhCL5S /tmp/tmp.LuISGKwzjA ++ return 0 + client_pod=pxc-client-56fd5498cd-7h2wc + wait_pod pxc-client-56fd5498cd-7h2wc + local pod=pxc-client-56fd5498cd-7h2wc + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-7h2wc ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-7h2wc condition met waiting for pod/pxc-client-56fd5498cd-7h2wc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.McPJo1bZq6/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.McPJo1bZq6/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.McPJo1bZq6/select-1.sql + is_keyring_plugin_in_use some-name + local cluster=some-name + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + kubectl exec some-name-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my + return 1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.secretsName}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.orbP2Mf8MJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.vjYOdN6hjJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.orbP2Mf8MJ ++ cat /tmp/tmp.vjYOdN6hjJ ++ rm /tmp/tmp.orbP2Mf8MJ /tmp/tmp.vjYOdN6hjJ ++ return 0 + secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.pqRUaG9QMS +++ mktemp ++ local LAST_ERR=/tmp/tmp.TYgGYmFex8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pqRUaG9QMS ++ cat /tmp/tmp.TYgGYmFex8 ++ rm /tmp/tmp.pqRUaG9QMS /tmp/tmp.TYgGYmFex8 ++ return 0 + root_pass=root_password + desc 'scale to 4 nodes to trigger SST' + set +o xtrace ----------------------------------------------------------------------------------- scale to 4 nodes to trigger SST ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type=merge -p '{"spec":{"pxc":{"size":4}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.1Zdexo0Mbk ++ mktemp + local LAST_ERR=/tmp/tmp.OI2ylC90cO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc some-name --type=merge -p '{"spec":{"pxc":{"size":4}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1Zdexo0Mbk perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.OI2ylC90cO + rm /tmp/tmp.1Zdexo0Mbk /tmp/tmp.OI2ylC90cO + return 0 + desc 'verify the joiner pod has SST retry support wired in' + set +o xtrace ----------------------------------------------------------------------------------- verify the joiner pod has SST retry support wired in ----------------------------------------------------------------------------------- + local retry + retry=0 + echo -n 'waiting for pod/some-name-pxc-3 container/pxc to start' waiting for pod/some-name-pxc-3 container/pxc to start+ local joiner_container_started ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.j8Z1VJKsg9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.6LiJK6tvi7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.j8Z1VJKsg9 ++ cat /tmp/tmp.6LiJK6tvi7 ++ rm /tmp/tmp.j8Z1VJKsg9 /tmp/tmp.6LiJK6tvi7 ++ return 0 + joiner_container_started= + [[ '' == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.hYlSlktZeM +++ mktemp ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' ++ local LAST_ERR=/tmp/tmp.KS3EwodYCL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hYlSlktZeM ++ cat /tmp/tmp.KS3EwodYCL ++ rm /tmp/tmp.hYlSlktZeM /tmp/tmp.KS3EwodYCL ++ return 0 + joiner_container_started= + [[ '' == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sdWdY0HqvH +++ mktemp ++ local LAST_ERR=/tmp/tmp.EhlYPQMWWY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sdWdY0HqvH ++ cat /tmp/tmp.EhlYPQMWWY ++ rm /tmp/tmp.sdWdY0HqvH /tmp/tmp.EhlYPQMWWY ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.u4QUWPMUzV +++ mktemp ++ local LAST_ERR=/tmp/tmp.GMs2Ibd2Sz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u4QUWPMUzV ++ cat /tmp/tmp.GMs2Ibd2Sz ++ rm /tmp/tmp.u4QUWPMUzV /tmp/tmp.GMs2Ibd2Sz ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EBqbiIyFD0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.YYtFtweHkk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EBqbiIyFD0 ++ cat /tmp/tmp.YYtFtweHkk ++ rm /tmp/tmp.EBqbiIyFD0 /tmp/tmp.YYtFtweHkk ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 240 ]] ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.LC4V27Z48L +++ mktemp ++ local LAST_ERR=/tmp/tmp.gouqR0xqyd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LC4V27Z48L ++ cat /tmp/tmp.gouqR0xqyd ++ rm /tmp/tmp.LC4V27Z48L /tmp/tmp.gouqR0xqyd ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.i3JJ360J2H +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZzqW37nvat ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.i3JJ360J2H ++ cat /tmp/tmp.ZzqW37nvat ++ rm /tmp/tmp.i3JJ360J2H /tmp/tmp.ZzqW37nvat ++ return 0 + joiner_container_started=true + [[ true == \t\r\u\e ]] + echo + desc 'interrupt SST twice so the joiner exhausts allowed retries' + set +o xtrace ----------------------------------------------------------------------------------- interrupt SST twice so the joiner exhausts allowed retries ----------------------------------------------------------------------------------- + interrupt_donor_during_sst root_password some-name-pxc-3 0 some-name-pxc-1 some-name-pxc-2 + local root_pass=root_password + local joiner_pod=some-name-pxc-3 + local current_restart_count=0 + local target_restart_count=1 + local donor_pod + donor_candidates=("${@:4}") + local donor_candidates + local retry=0 + local observed_restart_count + echo -n 'waiting for pod/some-name-pxc-3 restartCount=0 SST attempt' waiting for pod/some-name-pxc-3 restartCount=0 SST attempt+ : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=0 + [[ 0 == \0 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + kubectl_bin logs some-name-pxc-3 -c pxc + grep -q 'Waiting for SST streaming to complete!' + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] + : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=0 + [[ 0 == \0 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + echo + break ++ detect_active_donor root_password 120 some-name-pxc-1 some-name-pxc-2 ++ local root_pass=root_password ++ shift ++ local max_retry=120 ++ shift ++ local retry=0 ++ local donor_pod ++ local donor_rejects_queries ++ local local_state ++ echo -n 'waiting to detect active donor' waiting to detect active donor++ : ++ for donor_pod in '"$@"' +++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace +++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}' ++ donor_rejects_queries=OFF +++ awk '$1 == "wsrep_local_state_comment" {print $2}' +++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace ++ local_state=Donor/Desynced ++ [[ OFF == \O\N ]] ++ [[ Donor/Desynced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]] ++ echo some-name-pxc-1 ++ return 0 + donor_pod=some-name-pxc-1 + echo 'interrupting donor pod/some-name-pxc-1 during SST attempt 1' interrupting donor pod/some-name-pxc-1 during SST attempt 1 + kubectl_bin delete pod some-name-pxc-1 --wait=false --grace-period=0 --force ++ mktemp + local LAST_OUT=/tmp/tmp.Z6dGUhUgjH ++ mktemp + local LAST_ERR=/tmp/tmp.nEC6sP4bja + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod some-name-pxc-1 --wait=false --grace-period=0 --force + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Z6dGUhUgjH pod "some-name-pxc-1" force deleted from sst-retry-limit-11163 namespace + cat /tmp/tmp.nEC6sP4bja Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.Z6dGUhUgjH /tmp/tmp.nEC6sP4bja + return 0 + wait_for_pxc_restart_count some-name-pxc-3 1 600 + local pod=some-name-pxc-3 + local expected=1 + local max_retry=600 + local retry=0 + echo -n 'waiting for pod/some-name-pxc-3 restartCount to reach 1' waiting for pod/some-name-pxc-3 restartCount to reach 1++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=7 + [[ 7 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=8 + [[ 8 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=9 + [[ 9 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=10 + [[ 10 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=11 + [[ 11 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=12 + [[ 12 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=13 + [[ 13 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=14 + [[ 14 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=15 + [[ 15 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=16 + [[ 16 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=17 + [[ 17 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=18 + [[ 18 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=19 + [[ 19 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=20 + [[ 20 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=21 + [[ 21 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=22 + [[ 22 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=23 + [[ 23 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=24 + [[ 24 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=25 + [[ 25 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=26 + [[ 26 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=27 + [[ 27 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=28 + [[ 28 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=29 + [[ 29 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=30 + [[ 30 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=31 + [[ 31 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=32 + [[ 32 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=33 + [[ 33 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=34 + [[ 34 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=35 + [[ 35 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=36 + [[ 36 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=37 + [[ 37 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=38 + [[ 38 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=39 + [[ 39 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=40 + [[ 40 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=41 + [[ 41 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=42 + [[ 42 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=43 + [[ 43 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=44 + [[ 44 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=45 + [[ 45 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=46 + [[ 46 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=47 + [[ 47 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=48 + [[ 48 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=49 + [[ 49 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=50 + [[ 50 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=51 + [[ 51 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 1 ]] + echo + wait_pod some-name-pxc-1 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo some-name-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + interrupt_donor_during_sst root_password some-name-pxc-3 1 some-name-pxc-1 some-name-pxc-2 + local root_pass=root_password + local joiner_pod=some-name-pxc-3 + local current_restart_count=1 + local target_restart_count=2 + local donor_pod + donor_candidates=("${@:4}") + local donor_candidates + local retry=0 + local observed_restart_count + echo -n 'waiting for pod/some-name-pxc-3 restartCount=1 SST attempt' waiting for pod/some-name-pxc-3 restartCount=1 SST attempt+ : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=1 + [[ 1 == \1 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + echo + break ++ detect_active_donor root_password 120 some-name-pxc-1 some-name-pxc-2 ++ local root_pass=root_password ++ shift ++ local max_retry=120 ++ shift ++ local retry=0 ++ local donor_pod ++ local donor_rejects_queries ++ local local_state ++ echo -n 'waiting to detect active donor' waiting to detect active donor++ : ++ for donor_pod in '"$@"' +++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' +++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace ++ donor_rejects_queries=OFF +++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ awk '$1 == "wsrep_local_state_comment" {print $2}' +++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace ++ local_state=Donor/Desynced ++ [[ OFF == \O\N ]] ++ [[ Donor/Desynced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]] ++ echo some-name-pxc-1 ++ return 0 + donor_pod=some-name-pxc-1 + echo 'interrupting donor pod/some-name-pxc-1 during SST attempt 2' interrupting donor pod/some-name-pxc-1 during SST attempt 2 + kubectl_bin delete pod some-name-pxc-1 --wait=false --grace-period=0 --force ++ mktemp + local LAST_OUT=/tmp/tmp.0Kw9uaK2Px ++ mktemp + local LAST_ERR=/tmp/tmp.aPgaNtJBU6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod some-name-pxc-1 --wait=false --grace-period=0 --force + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0Kw9uaK2Px pod "some-name-pxc-1" force deleted from sst-retry-limit-11163 namespace + cat /tmp/tmp.aPgaNtJBU6 Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.0Kw9uaK2Px /tmp/tmp.aPgaNtJBU6 + return 0 + wait_for_pxc_restart_count some-name-pxc-3 2 600 + local pod=some-name-pxc-3 + local expected=2 + local max_retry=600 + local retry=0 + echo -n 'waiting for pod/some-name-pxc-3 restartCount to reach 2' waiting for pod/some-name-pxc-3 restartCount to reach 2++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=7 + [[ 7 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=8 + [[ 8 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=9 + [[ 9 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=10 + [[ 10 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=11 + [[ 11 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=12 + [[ 12 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=13 + [[ 13 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=14 + [[ 14 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=15 + [[ 15 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=16 + [[ 16 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=17 + [[ 17 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=18 + [[ 18 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=19 + [[ 19 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=20 + [[ 20 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=21 + [[ 21 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=22 + [[ 22 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=23 + [[ 23 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=24 + [[ 24 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=25 + [[ 25 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=26 + [[ 26 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=27 + [[ 27 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=28 + [[ 28 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=29 + [[ 29 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=30 + [[ 30 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=31 + [[ 31 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=32 + [[ 32 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=33 + [[ 33 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=34 + [[ 34 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=35 + [[ 35 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=36 + [[ 36 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=37 + [[ 37 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=38 + [[ 38 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=39 + [[ 39 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=40 + [[ 40 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=41 + [[ 41 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=42 + [[ 42 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=43 + [[ 43 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=44 + [[ 44 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=45 + [[ 45 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=46 + [[ 46 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=47 + [[ 47 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=48 + [[ 48 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=49 + [[ 49 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=50 + [[ 50 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=51 + [[ 51 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=52 + [[ 52 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=53 + [[ 53 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=54 + [[ 54 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=55 + [[ 55 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=56 + [[ 56 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=57 + [[ 57 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=58 + [[ 58 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=59 + [[ 59 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=60 + [[ 60 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=61 + [[ 61 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 2 -ge 2 ]] + echo + wait_pod some-name-pxc-1 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ echo some-name-pxc-1 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + desc 'verify the joiner stops restarting after the retry limit is reached' + set +o xtrace ----------------------------------------------------------------------------------- verify the joiner stops restarting after the retry limit is reached ----------------------------------------------------------------------------------- + retry=0 + echo -n 'waiting for pod/some-name-pxc-3 logs to contain '\''SST retry limit reached.'\''' waiting for pod/some-name-pxc-3 logs to contain 'SST retry limit reached.'+ grep -q 'SST retry limit reached.' + kubectl_bin logs some-name-pxc-3 -c pxc + echo + desc 'verify retries stop after the limit' + set +o xtrace ----------------------------------------------------------------------------------- verify retries stop after the limit ----------------------------------------------------------------------------------- + local joiner_phase ++ kubectl_bin get pod some-name-pxc-3 -o 'jsonpath={.status.phase}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SZI8k8GTH9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.wefnDQ7a52 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o 'jsonpath={.status.phase}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SZI8k8GTH9 ++ cat /tmp/tmp.wefnDQ7a52 ++ rm /tmp/tmp.SZI8k8GTH9 /tmp/tmp.wefnDQ7a52 ++ return 0 + joiner_phase=Running + local joiner_ready ++ jq -r '.status.conditions[] | select(.type == "Ready") | .status' ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.XoQ9Mt9cY0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.VPHJPIYRru ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XoQ9Mt9cY0 ++ cat /tmp/tmp.VPHJPIYRru ++ rm /tmp/tmp.XoQ9Mt9cY0 /tmp/tmp.VPHJPIYRru ++ return 0 + joiner_ready=False + [[ Running != \R\u\n\n\i\n\g ]] + [[ False != \F\a\l\s\e ]] + local restart_count_before ++ get_pxc_restart_count some-name-pxc-3 ++ local pod=some-name-pxc-3 ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[] | select(.name == "pxc") | .restartCount' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f6Y4fvJ7Vs +++ mktemp ++ local LAST_ERR=/tmp/tmp.t6Ua00QNfl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.f6Y4fvJ7Vs ++ cat /tmp/tmp.t6Ua00QNfl ++ rm /tmp/tmp.f6Y4fvJ7Vs /tmp/tmp.t6Ua00QNfl ++ return 0 + restart_count_before=2 + sleep_with_log 120 + local d=120 + log 'sleeping for 120 seconds' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-04-29T18:14:15+0000]' sleeping for 120 seconds [2026-04-29T18:14:15+0000] sleeping for 120 seconds + sleep 120 + local restart_count_after ++ get_pxc_restart_count some-name-pxc-3 ++ local pod=some-name-pxc-3 ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[] | select(.name == "pxc") | .restartCount' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IFRDzu4zC3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.jGCAbxC6Xe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IFRDzu4zC3 ++ cat /tmp/tmp.jGCAbxC6Xe ++ rm /tmp/tmp.IFRDzu4zC3 /tmp/tmp.jGCAbxC6Xe ++ return 0 + restart_count_after=2 + [[ 2 != \2 ]] + desc 'verify donor candidates remain healthy' + set +o xtrace ----------------------------------------------------------------------------------- verify donor candidates remain healthy ----------------------------------------------------------------------------------- + local donor_result + local donor_pod + for donor_pod in '"${donor_candidates[@]}"' ++ run_mysql_local 'SELECT COUNT(*) FROM myApp.myApp;' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 ++ local 'command=SELECT COUNT(*) FROM myApp.myApp;' ++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' ++ local pod=some-name-pxc-1 ++ local container_name= ++ set +o xtrace + donor_result=1 + [[ 1 != \1 ]] + for donor_pod in '"${donor_candidates[@]}"' ++ run_mysql_local 'SELECT COUNT(*) FROM myApp.myApp;' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-2 ++ local 'command=SELECT COUNT(*) FROM myApp.myApp;' ++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' ++ local pod=some-name-pxc-2 ++ local container_name= ++ set +o xtrace + donor_result=1 + [[ 1 != \1 ]] + destroy sst-retry-limit-11163 + local namespace=sst-retry-limit-11163 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v level=info + grep -v 'get backup status: Job.batch' + sort -u + tee /tmp/tmp.McPJo1bZq6/operator.log + grep -v 'the object has been modified' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ local LAST_OUT=/tmp/tmp.1xeJn0fiJn +++ mktemp ++ local LAST_ERR=/tmp/tmp.11mAYljFvH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1xeJn0fiJn ++ cat /tmp/tmp.11mAYljFvH ++ rm /tmp/tmp.1xeJn0fiJn /tmp/tmp.11mAYljFvH ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-7cd7bb76f4-kzxqr ++ mktemp + local LAST_OUT=/tmp/tmp.Q2lPi9lnqa ++ mktemp + local LAST_ERR=/tmp/tmp.f0gS5Kyac7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-7cd7bb76f4-kzxqr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Q2lPi9lnqa + cat /tmp/tmp.f0gS5Kyac7 + rm /tmp/tmp.Q2lPi9lnqa /tmp/tmp.f0gS5Kyac7 + return 0 } }, }, { }, }, { }, }, { }, - }, - { - }, + }, ... // 16 identical fields ... // 16 identical fields 2026-04-29T18:02:16.096Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-04-29T18:02:16.096Z INFO setup Manager starting up {"gitCommit": "9bec550e38b53ea8a42441eabf05aff8b718c288", "gitBranch": "PR-2433-9bec550e", "buildTime": "2026-04-29T14:21:52Z", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"} 2026-04-29T18:02:16.096Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.32.13-gke.1362000"} 2026-04-29T18:02:16.099Z INFO setup Registering Components. 2026-04-29T18:02:16.681Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-04-29T18:02:16.681Z INFO controller-runtime.metrics Starting metrics server 2026-04-29T18:02:16.681Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-04-29T18:02:16.681Z INFO setup Starting the Cmd. 2026-04-29T18:02:16.681Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-04-29T18:02:16.682Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-04-29T18:02:16.682Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-04-29T18:02:16.682Z INFO controller-runtime.webhook Starting webhook server 2026-04-29T18:02:16.683Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-04-29T18:02:16.783Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-04-29T18:02:16.808Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-04-29T18:02:16.809Z DEBUG events percona-xtradb-cluster-operator-7cd7bb76f4-kzxqr_7f68b14a-0c17-4350-aa60-d7549d8697f5 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"5af7f7b9-4f77-49dd-a1cf-46c7e7078e9c","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1777485736802703009"}, "reason": "LeaderElection"} 2026-04-29T18:02:16.809Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-04-29T18:02:16.809Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-04-29T18:02:16.809Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-04-29T18:02:16.809Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-04-29T18:02:16.909Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-04-29T18:02:16.909Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-04-29T18:02:16.909Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-04-29T18:02:16.909Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-04-29T18:02:16.910Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-04-29T18:02:16.910Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-04-29T18:03:15.388Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "32c5ee2d-080f-4e6a-850b-10ec5d4dd96d", "version": "1.20.0"} 2026-04-29T18:03:15.832Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "32c5ee2d-080f-4e6a-850b-10ec5d4dd96d", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-29T18:03:15.896Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "32c5ee2d-080f-4e6a-850b-10ec5d4dd96d", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-29T18:03:15.999Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "32c5ee2d-080f-4e6a-850b-10ec5d4dd96d", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-29T18:03:16.021Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "32c5ee2d-080f-4e6a-850b-10ec5d4dd96d", "object": "some-name-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-29T18:03:16.712Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "d0d3deae-7a3f-4c26-9060-5b143505b4ac", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-29T18:04:37.559Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "47b30331-c1b8-4878-9d9f-2e1dda3f7a2c", "user": "operator"} 2026-04-29T18:04:37.600Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "47b30331-c1b8-4878-9d9f-2e1dda3f7a2c", "user": "monitor"} 2026-04-29T18:04:37.659Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "47b30331-c1b8-4878-9d9f-2e1dda3f7a2c"} 2026-04-29T18:04:37.692Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "47b30331-c1b8-4878-9d9f-2e1dda3f7a2c"} 2026-04-29T18:04:37.725Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "47b30331-c1b8-4878-9d9f-2e1dda3f7a2c", "user": "xtrabackup"} 2026-04-29T18:04:37.774Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "47b30331-c1b8-4878-9d9f-2e1dda3f7a2c"} 2026-04-29T18:04:37.808Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "47b30331-c1b8-4878-9d9f-2e1dda3f7a2c", "user": "replication"} 2026-04-29T18:04:37.811Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "47b30331-c1b8-4878-9d9f-2e1dda3f7a2c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:04:42.896Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "835872f4-1176-4ea1-9345-f5fa2be9cf63", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:04:47.973Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "fa5246ed-943f-46c7-8c5b-1562ea2f6a59", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:04:53.070Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "f27926d9-c638-49ce-ae14-b2c6e75aeb20", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:04:58.150Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "12c4027f-94eb-443e-a5b5-06ec2556beb5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:05:03.258Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "25fce86c-1e87-4a84-a9e0-603dea18110b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:05:08.443Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "90b82d24-4cb5-4026-a53b-f2db58e79990", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:05:13.529Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "2abed7a7-ab6a-44ce-8a89-e0331af48f25", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:05:18.618Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "4545fdc7-fc46-4334-9a79-1d2d97920ece", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:05:23.692Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "c0be703c-c5ec-4a03-b754-c11ae7a1164d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:05:28.771Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "67522a9a-08ee-4909-a08a-7ec88a6372e0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:05:33.868Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "51a8d9f9-342c-4d12-8166-09ad5c8def66", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:05:38.959Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "96f74e14-48ef-4ecf-95a5-8095f7ac23cc", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:05:44.056Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "651a408c-63f8-449d-abe0-6f25fc555b68", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:05:49.138Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "c18fd4e4-1f52-416f-aec3-3190f2780029", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:05:54.224Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "6e40062a-6622-4b17-a7ad-92014541049d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:05:59.320Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "f3edad0b-3598-43ac-87f2-a9c2bc52f337", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:05:59.470Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "5530b2f9-b436-418b-8c98-8a69aeadf37d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:06:04.611Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "aed33366-eba6-4de5-be96-9d6ef26d27e0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:06:09.733Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "d1643fe0-b2df-496d-bb67-8334961dea41", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:06:14.851Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "0641ca82-9c36-4568-b063-34ebf2127713", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:06:19.977Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "bc31f03d-a1c7-4432-a387-15d78a070390", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:06:25.115Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "e55021c1-e85c-439f-9a74-2ccbc9cbd181", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:06:30.231Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "8baf857d-b169-4863-9f5b-208ed9ede7bd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:06:35.345Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "8d3ea7a1-2ae9-4408-9973-a4602e218a8d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:06:40.482Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "0fe14457-4651-4e6b-bd98-14482e0e1111", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:06:45.599Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "54713b2c-42fd-408b-9fa5-93832623492e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:06:50.741Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "1fc6f0e7-08fd-4347-b3af-bbc9fa52c3e2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:06:55.863Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "414933c6-1660-4ae1-904b-f7306246b399", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:00.988Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "63878387-a914-40ba-9b1c-722c2ad045c0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:06.119Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "5a5ad0b9-1278-447a-ac85-0a8bc47befe3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:11.253Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "bd78c8b2-dd66-42a6-a112-4991c0d81294", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:16.438Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "50a6babd-fda3-4b53-8dbf-14f98722a332", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:17.191Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "e69ad70a-03f7-444e-8cef-7b9a8a72e737", "user": "root"} 2026-04-29T18:07:17.212Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "e69ad70a-03f7-444e-8cef-7b9a8a72e737", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:17.215Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "e69ad70a-03f7-444e-8cef-7b9a8a72e737", "new version": "8.4.7-7.1"} 2026-04-29T18:07:17.989Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "e9152ad6-585b-4016-ae96-a056097785ca", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:23.681Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "d72cc873-3f30-4ea0-8f58-9e88c4ccd1c5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:28.846Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "b9c6c9c8-12c9-4512-9614-362982ac9885", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:34.026Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "5d759013-1843-4362-a4b2-d8e4c2ae46bd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:39.203Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "32eb0cbd-7b60-42d3-83e9-3ba3284afbb8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:44.374Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "ab24cf17-d8fd-4c5d-aee6-8f810e4c4787", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:49.550Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "f49999dc-8646-4a6f-9a3b-e76f25d8d666", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:54.717Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "3649b3d6-57cf-4042-b71f-077fa7c01946", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:07:59.927Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "9b368000-5dc8-4562-b2e3-f7c00ab81f24", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:08:05.299Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "e9c6ecad-27b4-42ca-ad46-65ea64b33416", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:08:10.489Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "e61909de-0032-41ea-8a03-e0cc25d08612", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:08:15.671Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "eab4e957-0fa7-41a9-beea-5cc19494c5b1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:08:20.849Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "41a598a6-b202-4b92-9c0f-54ae40b9159b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:08:26.032Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "648d5731-9309-408b-8721-5ccfea08a98f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:08:31.222Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "bb26f5df-be69-4709-8404-6446429b4322", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:08:36.414Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "6ffd070e-9688-45be-8a80-7c9deffb9647", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:08:41.608Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "253bcd38-657e-4e80-bec8-e868f6562e3a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:08:46.784Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "fc0e3bad-45a1-4511-9151-cc49e62f43b3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:08:51.936Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "7e28d700-541e-4b78-92b0-ca8a180ba165", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-04-29T18:08:51.985Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "7e28d700-541e-4b78-92b0-ca8a180ba165", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-04-29T18:08:52.065Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "7e28d700-541e-4b78-92b0-ca8a180ba165", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:08:52.792Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "cb083e1b-b9b5-49c1-ae07-f3dc5e7e14c2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:08:58.467Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "7e1b1686-ab73-4eab-9a27-c59c5f643e19", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:09:03.636Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "b39b1682-4aae-4444-a02b-55f776327109", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:09:08.804Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "14ebe1e4-4f3a-4a47-8a66-6cf9e8512ca2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:09:13.956Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "cedb2a6a-7793-409e-bddd-7becbc9a508e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:09:19.111Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "c2017734-fe83-41d5-a2a4-7fd75899ee80", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:09:24.277Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "162b1cbf-00d8-4fbf-b063-2d00278cb07d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:09:29.454Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "e87353c0-e169-4766-a427-3805d4527091", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:09:29.573Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "e9d287d2-84d4-42e3-b19a-1a4d21b1ff03", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:09:34.674Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "ddafcddc-b59a-4ca5-9c28-10004282b532", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:09:39.766Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "96b664cc-ec3d-4fcc-9a9c-ee541216ab96", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:09:44.877Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "33e4ff86-931f-4c2d-bc79-3b0883d21d03", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:09:49.982Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "4525ee96-78ac-4887-8550-6aaf6d8f7282", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:09:55.079Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "0a845d01-8b92-4b7b-a035-fe59e486ee47", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:10:00.182Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "5f557f55-d862-43f4-b353-416c0a686ce9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:10:05.320Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "e1aa37cd-df10-41aa-93a3-b5e816c51d55", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:10:10.420Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "80bbe05a-cf85-4cc3-91b3-5ee3b0115630", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:10:15.524Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "5753f250-bcf7-4ccc-a436-de0c473c5472", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:10:20.669Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "223eadcd-1f66-4e3b-a545-0ce39c87829c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:10:20.855Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "03bb923c-444d-4c5c-896c-1734339e387b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:10:26.511Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "eeeabd9f-f3d8-4f21-af63-5d3db97827be", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:10:31.662Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "e97c73fb-2380-4ff1-920c-778f4bff912f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:10:36.807Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "9cd5b7b6-e20f-42fb-8ac1-76cf4f7df5c0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:10:41.961Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "9c189406-8e13-4510-881b-3f9c513cfcaa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:10:47.141Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "20651533-db49-4480-a1d6-9b86d56f46c9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:10:52.294Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "117223e5-c001-4744-9ca0-1999b5930b37", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:10:57.461Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "55e02500-42c3-4cef-9939-ae07015432a5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:11:02.635Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "8823e562-d8d7-4937-bb93-223b853d63a0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:11:07.795Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "899107db-bea8-4471-81cb-940b316b1d33", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:11:12.979Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "64f7dfcb-af3d-408c-a7a1-448021df0397", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:11:18.129Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "9fc273c6-01e4-4577-9bde-dc140747cc5c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:11:23.306Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "3d56e9ba-cccb-4315-967e-1bf2a4e5be4d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:11:28.474Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "0abb18bb-4d7d-4bcc-8031-ea1a949c5bc9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:11:33.645Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "c793284d-48e7-4849-a2d1-a207dcf8b59c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:11:38.922Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "66ba299e-d729-40e9-b876-63ef4b0d8a6b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:11:44.249Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "8eebe0be-7893-4863-9d3f-6a93c741147c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:11:49.378Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "8d03997c-18ad-4ce2-a246-88db70d80fd5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:11:49.504Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "a3097263-e9db-492d-b28a-e8640be1fe19", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:11:54.598Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "9df50242-d762-49a5-9ce1-e1cb89bbd444", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:11:59.712Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "3db9bf1e-d7d4-4031-8eda-fd090b5a6a06", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:12:04.822Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "3366e055-0297-47a4-b773-8c7e7b08e4d5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:12:09.930Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "3beb4d2e-a850-463c-8f6a-97a59ffa444d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:12:15.049Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "8093671c-decd-4d3e-b5c4-21267e63485d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:12:20.172Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "c6ec49af-be4d-4732-89ab-8561889af59c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:12:25.282Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "5c94b330-c84e-4689-a50d-cf9491174160", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:12:30.385Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "a8ccde59-bd5c-4476-8976-136dd5df1d08", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:12:35.485Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "23ae20d5-623b-4197-87e6-ae810b202465", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:12:40.639Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "c3af8740-b44e-41e0-9e3f-ebae002b4c1a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:12:40.817Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "6fa2a042-c7a1-48be-ae30-5adf28a50a76", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:12:46.511Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "dad1ce3c-644e-4f03-9457-2e9efd4b9e00", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:12:51.684Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "06a94610-040c-4322-ae77-cb41226dd217", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:12:56.851Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "1dd04fe2-7c95-4566-860c-6db881352cf9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:13:02.022Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "b201a4a0-8fdb-4fde-9e2b-181b6856e5e4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:13:07.184Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "d35bb714-086b-46c7-bb47-4057692677b8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:13:12.336Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "935f8c67-e988-468a-8e1b-f42a63ea3b0e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:13:17.505Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "6cff7a52-90d1-4386-aea4-b8fb0a474ad0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:13:22.652Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "05788e0d-a70c-48c8-b597-0b097fb534b8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:13:27.805Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "286aa658-2ce0-452c-a3df-57f02acf1246", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:13:32.963Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "5c8c7453-0718-4d16-b63f-21462bf3699e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:13:38.109Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "3e1c4f68-a6dd-4c03-86ab-38af2018be23", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:13:43.298Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "ac068244-b6b7-4207-967f-09334216bd5b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:13:48.579Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "ce6aca44-ca68-43fc-8738-964725dd6dff", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:13:53.756Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "ede5b7f8-b077-48a0-a235-ea69ad4e4ab2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:13:53.933Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "3b513ab1-b9e7-465b-bb52-523cf9d6ff7b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:13:59.598Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "ace0a5e3-5419-40e5-944f-86a15b4de3fa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:14:04.752Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "d24976b1-b898-4d5e-b585-4c8c33939c76", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:14:09.896Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "81de9d0c-7568-426a-8b75-873d9c4eb326", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:14:10.064Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "1e6d45ee-ceff-4ebb-9b40-3375d179bd19", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:14:15.764Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "c4dbb453-7ae3-42f7-8c99-f570630ea5e1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:14:20.915Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "fba8f35c-af1d-400f-b7b7-6daf739bdd82", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:14:26.104Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "cbc64477-a13f-4076-a33b-731b5f5b3131", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:14:31.264Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "13fcbdda-649c-40da-8716-3690208d44e1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:14:36.429Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "a20292bd-a80a-453f-a541-3f417cc72c34", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:14:41.615Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "170ec729-e6d5-4cd6-afd9-328858e054a9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:14:46.779Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "162d51d5-7c01-4c18-add8-37ac9d480ea3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:14:51.945Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "48a90113-b6ee-45c7-a4db-eecd8b81c596", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:14:57.084Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "7ab2231f-db64-4cbd-a209-8c3a4c4b8069", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:15:02.237Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "47c9f6e4-4af5-418b-9f15-24ae44c52417", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:15:07.391Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "9360f7e3-d322-4a21-8b6d-ef78497f2513", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:15:12.569Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "2d292f9c-3144-4f29-998c-f9f003a83992", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:15:17.752Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "6008d1ed-ea7a-4460-86a7-f2c35ffb9f1a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:15:22.905Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "163e9b06-2a43-4b96-89e6-cd3f33bf1847", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:15:28.081Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "f3677b50-f915-41ed-96f0-bbcf4e7103e8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:15:33.245Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "527c9e59-275a-48f4-8244-ee1242807cdf", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:15:38.423Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "51d81126-b012-440c-9845-043d016feeff", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:15:43.570Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "25ff1916-c77f-46ba-89e4-9a6211c0f4da", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:15:48.770Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "13bc1893-eb01-4d00-a22b-ba5c8971c4fc", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:15:53.915Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "3bebceb2-3248-4a45-a05b-6280869239c4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:15:59.066Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "3c47d902-7032-4c70-ae2d-9b377ca2b35f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:16:04.225Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "1fb594b3-896d-4f61-a5d8-158433ceef29", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:16:09.385Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "ab211de2-3956-485e-9bca-91d281bf7b1a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:16:14.548Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "14efcc10-27a4-43af-8ff6-7dd18f0a3618", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-29T18:16:19.719Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-11163"}, "namespace": "sst-retry-limit-11163", "name": "some-name", "reconcileID": "5acafcfd-5379-4215-acbc-c377b4727d93", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} ... // 22 identical fields ... // 2 identical fields ... // 3 identical fields ... // 3 identical fields ... // 3 identical fields ... // 4 identical fields ... // 5 identical fields ... // 6 identical fields ... // 7 identical fields ... // 9 identical fields AccessModes: nil, ActiveDeadlineSeconds: nil, Affinity: nil, - Annotations: map[string]string{ + Annotations: map[string]string{ - APIVersion: "apps/v1", - APIVersion: "apps/v1", Args: {"mysqld"}, AutomountServiceAccountToken: nil, + AvailableReplicas: 0, - AvailableReplicas: 3, AWSElasticBlockStore: nil, AzureFile: nil, Capacity: nil, - CollisionCount: &0, + CollisionCount: nil, Conditions: nil, ConfigMap: &v1.ConfigMapVolumeSource{ ContainerPort: 3306, ContainerPort: 33060, ContainerPort: 33062, ContainerPort: 4444, ContainerPort: 4567, ContainerPort: 4568, Containers: []v1.Container{ + CreationTimestamp: v1.Time{}, - CreationTimestamp: v1.Time{Time: s"2026-04-29 18:03:15 +0000 UTC"}, + CurrentReplicas: 0, - CurrentReplicas: 3, + CurrentRevision: "", - CurrentRevision: "some-name-pxc-769576fbdf", DataSource: nil, DataSourceRef: nil, - DefaultMode: &420, - DefaultMode: &420, + DefaultMode: nil, + DefaultMode: nil, DeletionGracePeriodSeconds: nil, DeletionTimestamp: nil, + DeprecatedServiceAccount: "", - DeprecatedServiceAccount: "default", + DNSPolicy: "", - DNSPolicy: "ClusterFirst", EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "some-name-env-vars-pxc"}, Optional: &true}}}, Env: {{Name: "PXC_SERVICE", Value: "some-name-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name"}, Key: "xtrabackup"}}}, ...}, EphemeralContainers: nil, FailureThreshold: 3, FC: nil, - FieldsType: "FieldsV1", - FieldsType: "FieldsV1", - FieldsV1: s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`..., - FieldsV1: s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`..., Finalizers: nil, + Generation: 0, - Generation: 1, GitRepo: nil, HostAliases: nil, HostIP: "", HostPort: 0, ImagePullPolicy: "IfNotPresent", InitContainers: []v1.Container{ InitialDelaySeconds: 300, ISCSI: nil, Items: nil, Items: nil, Labels: nil, Lifecycle: nil, LivenessProbe: &v1.Probe{ LocalObjectReference: {Name: "auto-some-name-pxc"}, LocalObjectReference: {Name: "some-name-pxc"}, + ManagedFields: nil, - ManagedFields: []v1.ManagedFieldsEntry{ - Manager: "kube-controller-manager", - Manager: "percona-xtradb-cluster-operator", MinReadySeconds: 0, Name: "auto-config", {Name: "bin", VolumeSource: {EmptyDir: &{}}}, Name: "config", Name: "ist", Name: "mysql", Name: "mysql-admin", Name: "mysql-init-file", Name: "mysql-users-secret-file", Name: "mysqlx", Namespace: "sst-retry-limit-11163", Name: "ssl", Name: "ssl-internal", Name: "sst", {Name: "tmp", VolumeSource: {EmptyDir: &{}}}, Name: "vault-keyring-secret", Name: "write-set", NFS: nil, NodeName: "", NodeSelector: nil, ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "cfd6a52398268173b51d7cdb1331c09a", "percona.com/ssl-internal-hash": "751e59a1c1d94ac67f9d6bb6e756acdc"}}, ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, ObjectMeta: v1.ObjectMeta{ + ObservedGeneration: 0, - ObservedGeneration: 1, - Operation: "Update", - Operation: "Update", Optional: &false, Optional: &true, Optional: &true, Ordinals: nil, OS: nil, Overhead: nil, OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "some-name", UID: "b6fee400-0acc-428e-9397-a7ff0077768e", ...}}, - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJl"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6NCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJl"..., + PeriodSeconds: 0, - PeriodSeconds: 10, + PersistentVolumeClaimRetentionPolicy: nil, - PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}", + Phase: "", - Phase: "Pending", + PodManagementPolicy: "", - PodManagementPolicy: "OrderedReady", Ports: []v1.ContainerPort{ PreemptionPolicy: nil, ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}}, + Protocol: "", - Protocol: "TCP", Quobyte: nil, ReadinessProbe: &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...}, + ReadyReplicas: 0, - ReadyReplicas: 3, + Replicas: 0, - Replicas: 3, - Replicas: &3, + Replicas: &4, + ResourceVersion: "", - ResourceVersion: "1777486035125007003", + RestartPolicy: "", - RestartPolicy: "Always", - RevisionHistoryLimit: &10, + RevisionHistoryLimit: nil, + SchedulerName: "", - SchedulerName: "default-scheduler", SecretName: "internal-some-name", SecretName: "some-name-mysql-init", SecretName: "some-name-ssl", SecretName: "some-name-ssl-internal", SecretName: "some-name-vault", Secret: &v1.SecretVolumeSource{ SecurityContext: nil, Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, SelfLink: "", ServiceAccountName: "default", ServiceName: "some-name-pxc", SetHostnameAsFQDN: nil, Spec: v1.PersistentVolumeClaimSpec{ Spec: v1.PodSpec{ Spec: v1.StatefulSetSpec{ StartupProbe: nil, Status: v1.PersistentVolumeClaimStatus{ Status: v1.StatefulSetStatus{ StorageClassName: nil, Subdomain: "", - Subresource: "status", SuccessThreshold: 1, Template: v1.PodTemplateSpec{ TerminationGracePeriodSeconds: &600, TerminationGracePeriodSeconds: nil, + TerminationMessagePath: "", - TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "", - TerminationMessagePolicy: "File", TimeoutSeconds: 5, - Time: s"2026-04-29 18:03:15 +0000 UTC", - Time: s"2026-04-29 18:07:15 +0000 UTC", Tolerations: nil, - TopologySpreadConstraints: nil, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, TypeMeta: {}, TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"}, + UID: "", - UID: "84f58e31-d01e-4de7-9b93-6ee80a7731b4", + UpdatedReplicas: 0, - UpdatedReplicas: 3, + UpdateRevision: "", - UpdateRevision: "some-name-pxc-769576fbdf", UpdateStrategy: {Type: "RollingUpdate", RollingUpdate: &{Partition: &0}}, &v1.StatefulSet{ VolumeAttributesClassName: nil, VolumeClaimTemplates: []v1.PersistentVolumeClaim{ VolumeDevices: nil, - VolumeMode: &"Filesystem", + VolumeMode: nil, VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...}, VolumeName: "", VolumeSource: v1.VolumeSource{ Volumes: []v1.Volume{ VsphereVolume: nil, WorkingDir: "", + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n sst-retry-limit-11163 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.8R5PDOeS4J ++ mktemp + local LAST_ERR=/tmp/tmp.2KZWYkzUbd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8R5PDOeS4J perconaxtradbcluster.pxc.percona.com "some-name" deleted from sst-retry-limit-11163 namespace + cat /tmp/tmp.2KZWYkzUbd + rm /tmp/tmp.8R5PDOeS4J /tmp/tmp.2KZWYkzUbd + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.OIW40Sskhb ++ mktemp + local LAST_ERR=/tmp/tmp.aeZKaz25fo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OIW40Sskhb No resources found + cat /tmp/tmp.aeZKaz25fo + rm /tmp/tmp.OIW40Sskhb /tmp/tmp.aeZKaz25fo + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.VYe2ICKby5 ++ mktemp + local LAST_ERR=/tmp/tmp.z4vqvpqDr8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VYe2ICKby5 No resources found + cat /tmp/tmp.z4vqvpqDr8 + rm /tmp/tmp.VYe2ICKby5 /tmp/tmp.z4vqvpqDr8 + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.l4OvOZ7zKC ++ mktemp + local LAST_ERR=/tmp/tmp.RqsxfYpYI6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.l4OvOZ7zKC validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.RqsxfYpYI6 + rm /tmp/tmp.l4OvOZ7zKC /tmp/tmp.RqsxfYpYI6 + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + rm -rf /tmp/tmp.McPJo1bZq6 + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator + kubectl_bin delete --grace-period=0 --force=true namespace sst-retry-limit-11163 ++ mktemp ++ mktemp + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.c2YpQoZXEc + local LAST_OUT=/tmp/tmp.f3KUdYfWbK ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.Vjam55Jul2 + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.UpidoA10T0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace sst-retry-limit-11163