Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/logs/sst-retry-limit-8-0.log Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1 + main + local cluster=some-name + local joiner_pod=some-name-pxc-3 + donor_candidates=("${cluster}-pxc-1" "${cluster}-pxc-2") + local donor_candidates + create_infra sst-retry-limit-22407 + local ns=sst-retry-limit-22407 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl patch pxc -n sst-retry-limit-17738 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.rY6CrEHKdP ++ mktemp + local LAST_ERR=/tmp/tmp.LRhqFocBHc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rY6CrEHKdP perconaxtradbcluster.pxc.percona.com "some-name" deleted from sst-retry-limit-17738 namespace + cat /tmp/tmp.LRhqFocBHc + rm /tmp/tmp.rY6CrEHKdP /tmp/tmp.LRhqFocBHc + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.1AFeG3nCZv ++ mktemp + local LAST_ERR=/tmp/tmp.VIgL8kA8Sa + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1AFeG3nCZv No resources found + cat /tmp/tmp.VIgL8kA8Sa + rm /tmp/tmp.1AFeG3nCZv /tmp/tmp.VIgL8kA8Sa + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.ZXVkkpIFBZ ++ mktemp + local LAST_ERR=/tmp/tmp.sffFmkWaMu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZXVkkpIFBZ No resources found + cat /tmp/tmp.sffFmkWaMu + rm /tmp/tmp.ZXVkkpIFBZ /tmp/tmp.sffFmkWaMu + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' ++ mktemp + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.wWTK0CLe15 + local LAST_OUT=/tmp/tmp.BlSZ14Py5F ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.bzcKTOMImQ + local exit_status=0 + local LAST_ERR=/tmp/tmp.RAwh1mHlVJ + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + for i in '$(seq 0 2)' + kubectl delete namespace pxc-operator + set +e + kubectl get ns + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BlSZ14Py5F + cat /tmp/tmp.RAwh1mHlVJ + rm /tmp/tmp.BlSZ14Py5F /tmp/tmp.RAwh1mHlVJ + return 0 namespace "sst-retry-limit-17738" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wWTK0CLe15 namespace "pxc-operator" deleted + cat /tmp/tmp.bzcKTOMImQ + rm /tmp/tmp.wWTK0CLe15 /tmp/tmp.bzcKTOMImQ + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.NXWg1t6TPV ++ mktemp + local LAST_ERR=/tmp/tmp.tABQSEBUIL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NXWg1t6TPV namespace/pxc-operator created + cat /tmp/tmp.tABQSEBUIL + rm /tmp/tmp.NXWg1t6TPV /tmp/tmp.tABQSEBUIL + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.SfUMxq7Y5A +++ mktemp ++ local LAST_ERR=/tmp/tmp.U7SdfIqURS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SfUMxq7Y5A ++ cat /tmp/tmp.U7SdfIqURS ++ rm /tmp/tmp.SfUMxq7Y5A /tmp/tmp.U7SdfIqURS ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2449-b47912f7-4-cluster2 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.TOldJCBWtI ++ mktemp + local LAST_ERR=/tmp/tmp.MuruCZqfCx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2449-b47912f7-4-cluster2 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TOldJCBWtI Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2449-b47912f7-4-cluster2" modified. + cat /tmp/tmp.MuruCZqfCx + rm /tmp/tmp.TOldJCBWtI /tmp/tmp.MuruCZqfCx + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.0xzzKNMAXk ++ mktemp + local LAST_ERR=/tmp/tmp.EIKYDOF6l5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0xzzKNMAXk customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.EIKYDOF6l5 + rm /tmp/tmp.0xzzKNMAXk /tmp/tmp.EIKYDOF6l5 + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/deploy/cw-rbac.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.2qFjjs71dV ++ mktemp + local LAST_ERR=/tmp/tmp.G0oSqjzYvP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2qFjjs71dV clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.G0oSqjzYvP + rm /tmp/tmp.2qFjjs71dV /tmp/tmp.G0oSqjzYvP + return 0 + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/deploy/cw-operator.yaml + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + kubectl_bin apply -f - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2449-b47912f7^' ++ mktemp + local LAST_OUT=/tmp/tmp.njAxyx6nTS ++ mktemp + local LAST_ERR=/tmp/tmp.lr1DWYoVcN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.njAxyx6nTS deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.lr1DWYoVcN + rm /tmp/tmp.njAxyx6nTS /tmp/tmp.lr1DWYoVcN + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.6OLJJf25Ap ++ mktemp + local LAST_ERR=/tmp/tmp.dq3S7W6znE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6OLJJf25Ap pod/percona-xtradb-cluster-operator-7d9c4fb94c-ghkzd condition met + cat /tmp/tmp.dq3S7W6znE + rm /tmp/tmp.6OLJJf25Ap /tmp/tmp.dq3S7W6znE + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.okfhuQ4vNp +++ mktemp ++ local LAST_ERR=/tmp/tmp.qHjKe4icxR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.okfhuQ4vNp ++ cat /tmp/tmp.qHjKe4icxR ++ rm /tmp/tmp.okfhuQ4vNp /tmp/tmp.qHjKe4icxR ++ return 0 + wait_pod percona-xtradb-cluster-operator-7d9c4fb94c-ghkzd 480 pxc-operator + local pod=percona-xtradb-cluster-operator-7d9c4fb94c-ghkzd + local max_retry=480 + local ns=pxc-operator ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo percona-xtradb-cluster-operator-7d9c4fb94c-ghkzd + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-7d9c4fb94c-ghkzd condition met waiting for pod/percona-xtradb-cluster-operator-7d9c4fb94c-ghkzd to become Ready.Ok + sleep 3 + create_namespace sst-retry-limit-22407 + local namespace=sst-retry-limit-22407 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ tail -n1 ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces sst-retry-limit-22407' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces sst-retry-limit-22407 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace sst-retry-limit-22407 + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + xargs kubectl delete ns + awk '{print$1}' + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.Qso8mOrNx8 + local LAST_OUT=/tmp/tmp.GiH9CBLDDX ++ mktemp + local LAST_ERR=/tmp/tmp.YtZHEMx5NH + local exit_status=0 ++ seq 0 2 ++ mktemp + local LAST_ERR=/tmp/tmp.Uymarzgfut + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace sst-retry-limit-22407 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Qso8mOrNx8 + cat /tmp/tmp.YtZHEMx5NH + rm /tmp/tmp.Qso8mOrNx8 /tmp/tmp.YtZHEMx5NH + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace sst-retry-limit-22407 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace sst-retry-limit-22407 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.GiH9CBLDDX + cat /tmp/tmp.Uymarzgfut Error from server (NotFound): namespaces "sst-retry-limit-22407" not found + rm /tmp/tmp.GiH9CBLDDX /tmp/tmp.Uymarzgfut + return 1 + : + wait_for_delete namespace/sst-retry-limit-22407 + local res=namespace/sst-retry-limit-22407 + echo -n 'waiting for namespace/sst-retry-limit-22407 to be deleted' waiting for namespace/sst-retry-limit-22407 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "sst-retry-limit-22407" not found + desc 'create namespace sst-retry-limit-22407' + set +o xtrace ----------------------------------------------------------------------------------- create namespace sst-retry-limit-22407 ----------------------------------------------------------------------------------- + kubectl_bin create namespace sst-retry-limit-22407 ++ mktemp + local LAST_OUT=/tmp/tmp.y3xGyieM4P ++ mktemp + local LAST_ERR=/tmp/tmp.7KQDevBv7d + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace sst-retry-limit-22407 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.y3xGyieM4P namespace/sst-retry-limit-22407 created + cat /tmp/tmp.7KQDevBv7d + rm /tmp/tmp.y3xGyieM4P /tmp/tmp.7KQDevBv7d + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.0tHAooHc7Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.qYHRSCtdHN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0tHAooHc7Q ++ cat /tmp/tmp.qYHRSCtdHN ++ rm /tmp/tmp.0tHAooHc7Q /tmp/tmp.qYHRSCtdHN ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2449-b47912f7-4-cluster2 --namespace=sst-retry-limit-22407 ++ mktemp + local LAST_OUT=/tmp/tmp.dOtZJQ9Si0 ++ mktemp + local LAST_ERR=/tmp/tmp.FBH4maWAo9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2449-b47912f7-4-cluster2 --namespace=sst-retry-limit-22407 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dOtZJQ9Si0 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2449-b47912f7-4-cluster2" modified. + cat /tmp/tmp.FBH4maWAo9 + rm /tmp/tmp.dOtZJQ9Si0 /tmp/tmp.FBH4maWAo9 + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xgtP0ZD1tI ++ mktemp + local LAST_ERR=/tmp/tmp.oqmOk7ae9v + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xgtP0ZD1tI secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.oqmOk7ae9v + rm /tmp/tmp.xgtP0ZD1tI /tmp/tmp.oqmOk7ae9v + return 0 + desc 'start 3-node cluster with SST retry limit' + set +o xtrace ----------------------------------------------------------------------------------- start 3-node cluster with SST retry limit ----------------------------------------------------------------------------------- + spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml 3 10 /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/secrets.yml /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.FtvKVpKb51 ++ mktemp + local LAST_ERR=/tmp/tmp.ansk5BBSMw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FtvKVpKb51 secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.ansk5BBSMw + rm /tmp/tmp.FtvKVpKb51 /tmp/tmp.ansk5BBSMw + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml + local pvc_name= + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.rRRPg2cpUd + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2449-b47912f7#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.sst-retry-limit-22407~ + local LAST_ERR=/tmp/tmp.rZPwRDBysk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#apply:.*#apply: Never#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rRRPg2cpUd deployment.apps/pxc-client created + cat /tmp/tmp.rZPwRDBysk + rm /tmp/tmp.rRRPg2cpUd /tmp/tmp.rZPwRDBysk + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2449-b47912f7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.sst-retry-limit-22407~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.2Gs2akOCkP ++ mktemp + local LAST_ERR=/tmp/tmp.x1PWIhVFJk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2Gs2akOCkP perconaxtradbcluster.pxc.percona.com/some-name created + cat /tmp/tmp.x1PWIhVFJk + rm /tmp/tmp.2Gs2akOCkP /tmp/tmp.x1PWIhVFJk + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jxd5suKqwM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cjcivMGAsA +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.jxd5suKqwM +++ cat /tmp/tmp.cjcivMGAsA +++ rm /tmp/tmp.jxd5suKqwM /tmp/tmp.cjcivMGAsA +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VNBYdEt4MQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yKSzUMOQEu +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.VNBYdEt4MQ +++ cat /tmp/tmp.yKSzUMOQEu +++ rm /tmp/tmp.VNBYdEt4MQ /tmp/tmp.yKSzUMOQEu +++ return 0 ++ [[ false == \t\r\u\e ]] ++ echo some-name-pxc + local proxy=some-name-pxc + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-22407 ++ mktemp + local LAST_OUT=/tmp/tmp.wmUcT8tbVU ++ mktemp + local LAST_ERR=/tmp/tmp.BQiok4fz4e + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-22407 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-22407 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-22407 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.wmUcT8tbVU + cat /tmp/tmp.BQiok4fz4e error: no matching resources found + rm /tmp/tmp.wmUcT8tbVU /tmp/tmp.BQiok4fz4e + return 1 + true + wait_for_running some-name-pxc 1 + local name=some-name-pxc + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo some-name-pxc-0 + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met waiting for pod/some-name-pxc-0 to become Ready.Ok + wait_for_running some-name-pxc 3 + local name=some-name-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ echo some-name-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met waiting for pod/some-name-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-1 480 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo some-name-pxc-1 + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-2 480 + local pod=some-name-pxc-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo some-name-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-pxc-2 condition met waiting for pod/some-name-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.pGjoPdHgeq +++ mktemp ++ local LAST_ERR=/tmp/tmp.Fxu38aHD87 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pGjoPdHgeq ++ cat /tmp/tmp.Fxu38aHD87 ++ rm /tmp/tmp.pGjoPdHgeq /tmp/tmp.Fxu38aHD87 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RPidIfP4L3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.8SeCj7E7ds ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RPidIfP4L3 ++ cat /tmp/tmp.8SeCj7E7ds ++ rm /tmp/tmp.RPidIfP4L3 /tmp/tmp.8SeCj7E7ds ++ return 0 + client_pod=pxc-client-67fc4995bb-b4whd + wait_pod pxc-client-67fc4995bb-b4whd + local pod=pxc-client-67fc4995bb-b4whd + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-b4whd ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-b4whd condition met waiting for pod/pxc-client-67fc4995bb-b4whd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wjUmsh9Fxd +++ mktemp ++ local LAST_ERR=/tmp/tmp.BjHe8z9Kew ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wjUmsh9Fxd ++ cat /tmp/tmp.BjHe8z9Kew ++ rm /tmp/tmp.wjUmsh9Fxd /tmp/tmp.BjHe8z9Kew ++ return 0 + client_pod=pxc-client-67fc4995bb-b4whd + wait_pod pxc-client-67fc4995bb-b4whd + local pod=pxc-client-67fc4995bb-b4whd + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-b4whd ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-b4whd condition met waiting for pod/pxc-client-67fc4995bb-b4whd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.53cfsDqBHJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.5HhOuls9yn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.53cfsDqBHJ ++ cat /tmp/tmp.5HhOuls9yn ++ rm /tmp/tmp.53cfsDqBHJ /tmp/tmp.5HhOuls9yn ++ return 0 + client_pod=pxc-client-67fc4995bb-b4whd + wait_pod pxc-client-67fc4995bb-b4whd + local pod=pxc-client-67fc4995bb-b4whd + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-b4whd ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-b4whd condition met waiting for pod/pxc-client-67fc4995bb-b4whd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.X6S66n74BV/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.X6S66n74BV/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.X6S66n74BV/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2G4rpfGDz7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.TxvrWXcjML ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2G4rpfGDz7 ++ cat /tmp/tmp.TxvrWXcjML ++ rm /tmp/tmp.2G4rpfGDz7 /tmp/tmp.TxvrWXcjML ++ return 0 + client_pod=pxc-client-67fc4995bb-b4whd + wait_pod pxc-client-67fc4995bb-b4whd + local pod=pxc-client-67fc4995bb-b4whd + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-b4whd ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-b4whd condition met waiting for pod/pxc-client-67fc4995bb-b4whd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.X6S66n74BV/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.X6S66n74BV/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.X6S66n74BV/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HYw05MZqEd +++ mktemp ++ local LAST_ERR=/tmp/tmp.g8uBahnKmM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HYw05MZqEd ++ cat /tmp/tmp.g8uBahnKmM ++ rm /tmp/tmp.HYw05MZqEd /tmp/tmp.g8uBahnKmM ++ return 0 + client_pod=pxc-client-67fc4995bb-b4whd + wait_pod pxc-client-67fc4995bb-b4whd + local pod=pxc-client-67fc4995bb-b4whd + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-b4whd ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-b4whd condition met waiting for pod/pxc-client-67fc4995bb-b4whd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.X6S66n74BV/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.X6S66n74BV/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.X6S66n74BV/select-1.sql + is_keyring_plugin_in_use some-name + local cluster=some-name + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + kubectl exec some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + grep -E -o 'early-plugin-load=keyring_\w+.so' + return 1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.secretsName}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iGzb7dOyCY +++ mktemp ++ local LAST_ERR=/tmp/tmp.JHHi1wPVnq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iGzb7dOyCY ++ cat /tmp/tmp.JHHi1wPVnq ++ rm /tmp/tmp.iGzb7dOyCY /tmp/tmp.JHHi1wPVnq ++ return 0 + secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FpObGqDyYX +++ mktemp ++ local LAST_ERR=/tmp/tmp.r4dUvhfnQs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FpObGqDyYX ++ cat /tmp/tmp.r4dUvhfnQs ++ rm /tmp/tmp.FpObGqDyYX /tmp/tmp.r4dUvhfnQs ++ return 0 + root_pass=root_password + desc 'scale to 4 nodes to trigger SST' + set +o xtrace ----------------------------------------------------------------------------------- scale to 4 nodes to trigger SST ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type=merge -p '{"spec":{"pxc":{"size":4}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.49n1wSQYMn ++ mktemp + local LAST_ERR=/tmp/tmp.sZICGyWM2P + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc some-name --type=merge -p '{"spec":{"pxc":{"size":4}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.49n1wSQYMn perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.sZICGyWM2P + rm /tmp/tmp.49n1wSQYMn /tmp/tmp.sZICGyWM2P + return 0 + desc 'verify the joiner pod has SST retry support wired in' + set +o xtrace ----------------------------------------------------------------------------------- verify the joiner pod has SST retry support wired in ----------------------------------------------------------------------------------- + local retry + retry=0 + echo -n 'waiting for pod/some-name-pxc-3 container/pxc to start' waiting for pod/some-name-pxc-3 container/pxc to start+ local joiner_container_started ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.J0dXlnvFCH +++ mktemp ++ local LAST_ERR=/tmp/tmp.E8cM6Bwh9j ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.J0dXlnvFCH ++ cat /tmp/tmp.E8cM6Bwh9j ++ rm /tmp/tmp.J0dXlnvFCH /tmp/tmp.E8cM6Bwh9j ++ return 0 + joiner_container_started= + [[ '' == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 240 ]] ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.rGBlTcbiUK +++ mktemp ++ local LAST_ERR=/tmp/tmp.7LroJpGDQi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rGBlTcbiUK ++ cat /tmp/tmp.7LroJpGDQi ++ rm /tmp/tmp.rGBlTcbiUK /tmp/tmp.7LroJpGDQi ++ return 0 + joiner_container_started= + [[ '' == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8kmGA0DpOb +++ mktemp ++ local LAST_ERR=/tmp/tmp.kS3y8Fihaa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8kmGA0DpOb ++ cat /tmp/tmp.kS3y8Fihaa ++ rm /tmp/tmp.8kmGA0DpOb /tmp/tmp.kS3y8Fihaa ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.O0kdYcRo5x +++ mktemp ++ local LAST_ERR=/tmp/tmp.AgUiknKlwl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.O0kdYcRo5x ++ cat /tmp/tmp.AgUiknKlwl ++ rm /tmp/tmp.O0kdYcRo5x /tmp/tmp.AgUiknKlwl ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BTBUbTzTz3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.vpXHPxJEqo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BTBUbTzTz3 ++ cat /tmp/tmp.vpXHPxJEqo ++ rm /tmp/tmp.BTBUbTzTz3 /tmp/tmp.vpXHPxJEqo ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hYjwJot3MO +++ mktemp ++ local LAST_ERR=/tmp/tmp.CBuS5ZDNo5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hYjwJot3MO ++ cat /tmp/tmp.CBuS5ZDNo5 ++ rm /tmp/tmp.hYjwJot3MO /tmp/tmp.CBuS5ZDNo5 ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7efOEEcwax +++ mktemp ++ local LAST_ERR=/tmp/tmp.JnK7kMssME ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7efOEEcwax ++ cat /tmp/tmp.JnK7kMssME ++ rm /tmp/tmp.7efOEEcwax /tmp/tmp.JnK7kMssME ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=7 + [[ 7 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8xdUZEGjhh +++ mktemp ++ local LAST_ERR=/tmp/tmp.DiPpGx00fI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8xdUZEGjhh ++ cat /tmp/tmp.DiPpGx00fI ++ rm /tmp/tmp.8xdUZEGjhh /tmp/tmp.DiPpGx00fI ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=8 + [[ 8 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.T5R3ajYJI2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.cK2nNj2sYL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.T5R3ajYJI2 ++ cat /tmp/tmp.cK2nNj2sYL ++ rm /tmp/tmp.T5R3ajYJI2 /tmp/tmp.cK2nNj2sYL ++ return 0 + joiner_container_started=true + [[ true == \t\r\u\e ]] + echo + desc 'interrupt SST twice so the joiner exhausts allowed retries' + set +o xtrace ----------------------------------------------------------------------------------- interrupt SST twice so the joiner exhausts allowed retries ----------------------------------------------------------------------------------- + interrupt_donor_during_sst root_password some-name-pxc-3 0 some-name-pxc-1 some-name-pxc-2 + local root_pass=root_password + local joiner_pod=some-name-pxc-3 + local current_restart_count=0 + local target_restart_count=1 + local donor_pod + donor_candidates=("${@:4}") + local donor_candidates + local retry=0 + local observed_restart_count + echo -n 'waiting for pod/some-name-pxc-3 restartCount=0 SST attempt' waiting for pod/some-name-pxc-3 restartCount=0 SST attempt+ : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=0 + [[ 0 == \0 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + kubectl_bin logs some-name-pxc-3 -c pxc + grep -q 'Waiting for SST streaming to complete!' + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] + : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=0 + [[ 0 == \0 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + echo + break ++ detect_active_donor root_password 120 some-name-pxc-1 some-name-pxc-2 ++ local root_pass=root_password ++ shift ++ local max_retry=120 ++ shift ++ local retry=0 ++ local donor_pod ++ local donor_rejects_queries ++ local local_state ++ echo -n 'waiting to detect active donor' waiting to detect active donor++ : ++ for donor_pod in '"$@"' +++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}' +++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace ++ donor_rejects_queries=OFF +++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace +++ awk '$1 == "wsrep_local_state_comment" {print $2}' ++ local_state=Donor/Desynced ++ [[ OFF == \O\N ]] ++ [[ Donor/Desynced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]] ++ echo some-name-pxc-1 ++ return 0 + donor_pod=some-name-pxc-1 + echo 'interrupting donor pod/some-name-pxc-1 during SST attempt 1' interrupting donor pod/some-name-pxc-1 during SST attempt 1 + kubectl_bin delete pod some-name-pxc-1 --wait=false --grace-period=0 --force ++ mktemp + local LAST_OUT=/tmp/tmp.s3TJYtCWKw ++ mktemp + local LAST_ERR=/tmp/tmp.GKcK34GUy5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod some-name-pxc-1 --wait=false --grace-period=0 --force + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.s3TJYtCWKw pod "some-name-pxc-1" force deleted from sst-retry-limit-22407 namespace + cat /tmp/tmp.GKcK34GUy5 Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.s3TJYtCWKw /tmp/tmp.GKcK34GUy5 + return 0 + wait_for_pxc_restart_count some-name-pxc-3 1 600 + local pod=some-name-pxc-3 + local expected=1 + local max_retry=600 + local retry=0 + echo -n 'waiting for pod/some-name-pxc-3 restartCount to reach 1' waiting for pod/some-name-pxc-3 restartCount to reach 1++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=7 + [[ 7 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=8 + [[ 8 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=9 + [[ 9 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=10 + [[ 10 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=11 + [[ 11 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=12 + [[ 12 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=13 + [[ 13 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=14 + [[ 14 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=15 + [[ 15 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=16 + [[ 16 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=17 + [[ 17 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=18 + [[ 18 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=19 + [[ 19 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=20 + [[ 20 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=21 + [[ 21 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=22 + [[ 22 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=23 + [[ 23 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=24 + [[ 24 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=25 + [[ 25 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=26 + [[ 26 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=27 + [[ 27 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=28 + [[ 28 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=29 + [[ 29 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=30 + [[ 30 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=31 + [[ 31 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=32 + [[ 32 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=33 + [[ 33 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=34 + [[ 34 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=35 + [[ 35 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=36 + [[ 36 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=37 + [[ 37 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=38 + [[ 38 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=39 + [[ 39 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=40 + [[ 40 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=41 + [[ 41 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=42 + [[ 42 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=43 + [[ 43 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=44 + [[ 44 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=45 + [[ 45 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=46 + [[ 46 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=47 + [[ 47 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=48 + [[ 48 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 1 ]] + echo + wait_pod some-name-pxc-1 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ echo some-name-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + interrupt_donor_during_sst root_password some-name-pxc-3 1 some-name-pxc-1 some-name-pxc-2 + local root_pass=root_password + local joiner_pod=some-name-pxc-3 + local current_restart_count=1 + local target_restart_count=2 + local donor_pod + donor_candidates=("${@:4}") + local donor_candidates + local retry=0 + local observed_restart_count + echo -n 'waiting for pod/some-name-pxc-3 restartCount=1 SST attempt' waiting for pod/some-name-pxc-3 restartCount=1 SST attempt+ : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=1 + [[ 1 == \1 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + echo + break ++ detect_active_donor root_password 120 some-name-pxc-1 some-name-pxc-2 ++ local root_pass=root_password ++ shift ++ local max_retry=120 ++ shift ++ local retry=0 ++ local donor_pod ++ local donor_rejects_queries ++ local local_state ++ echo -n 'waiting to detect active donor' waiting to detect active donor++ : ++ for donor_pod in '"$@"' +++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace +++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}' ++ donor_rejects_queries=OFF +++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace +++ awk '$1 == "wsrep_local_state_comment" {print $2}' ++ local_state=Synced ++ [[ OFF == \O\N ]] ++ [[ Synced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]] ++ for donor_pod in '"$@"' +++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-2 +++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-2 +++ local container_name= +++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}' +++ set +o xtrace ++ donor_rejects_queries=OFF +++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-2 +++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-2 +++ awk '$1 == "wsrep_local_state_comment" {print $2}' +++ local container_name= +++ set +o xtrace ++ local_state=Synced ++ [[ OFF == \O\N ]] ++ [[ Synced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]] ++ sleep 1 ++ echo -n . .++ retry=1 ++ [[ 1 -ge 120 ]] ++ : ++ for donor_pod in '"$@"' +++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace +++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}' ++ donor_rejects_queries=OFF +++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ awk '$1 == "wsrep_local_state_comment" {print $2}' +++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace ++ local_state=Donor/Desynced ++ [[ OFF == \O\N ]] ++ [[ Donor/Desynced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]] ++ echo some-name-pxc-1 ++ return 0 + donor_pod=some-name-pxc-1 + echo 'interrupting donor pod/some-name-pxc-1 during SST attempt 2' interrupting donor pod/some-name-pxc-1 during SST attempt 2 + kubectl_bin delete pod some-name-pxc-1 --wait=false --grace-period=0 --force ++ mktemp + local LAST_OUT=/tmp/tmp.CYfbVgBPlH ++ mktemp + local LAST_ERR=/tmp/tmp.gPZF4eLAQn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod some-name-pxc-1 --wait=false --grace-period=0 --force + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CYfbVgBPlH pod "some-name-pxc-1" force deleted from sst-retry-limit-22407 namespace + cat /tmp/tmp.gPZF4eLAQn Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.CYfbVgBPlH /tmp/tmp.gPZF4eLAQn + return 0 + wait_for_pxc_restart_count some-name-pxc-3 2 600 + local pod=some-name-pxc-3 + local expected=2 + local max_retry=600 + local retry=0 + echo -n 'waiting for pod/some-name-pxc-3 restartCount to reach 2' waiting for pod/some-name-pxc-3 restartCount to reach 2++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=7 + [[ 7 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=8 + [[ 8 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=9 + [[ 9 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=10 + [[ 10 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=11 + [[ 11 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=12 + [[ 12 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=13 + [[ 13 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=14 + [[ 14 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=15 + [[ 15 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=16 + [[ 16 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=17 + [[ 17 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=18 + [[ 18 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=19 + [[ 19 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=20 + [[ 20 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=21 + [[ 21 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=22 + [[ 22 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=23 + [[ 23 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=24 + [[ 24 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=25 + [[ 25 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=26 + [[ 26 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=27 + [[ 27 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=28 + [[ 28 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=29 + [[ 29 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=30 + [[ 30 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=31 + [[ 31 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=32 + [[ 32 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=33 + [[ 33 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=34 + [[ 34 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=35 + [[ 35 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=36 + [[ 36 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=37 + [[ 37 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=38 + [[ 38 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=39 + [[ 39 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=40 + [[ 40 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=41 + [[ 41 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=42 + [[ 42 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=43 + [[ 43 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=44 + [[ 44 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=45 + [[ 45 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=46 + [[ 46 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=47 + [[ 47 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=48 + [[ 48 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=49 + [[ 49 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=50 + [[ 50 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=51 + [[ 51 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=52 + [[ 52 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=53 + [[ 53 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=54 + [[ 54 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=55 + [[ 55 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=56 + [[ 56 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=57 + [[ 57 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=58 + [[ 58 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=59 + [[ 59 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 2 -ge 2 ]] + echo + wait_pod some-name-pxc-1 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ echo some-name-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + desc 'verify the joiner stops restarting after the retry limit is reached' + set +o xtrace ----------------------------------------------------------------------------------- verify the joiner stops restarting after the retry limit is reached ----------------------------------------------------------------------------------- + retry=0 + echo -n 'waiting for pod/some-name-pxc-3 logs to contain '\''SST retry limit reached.'\''' waiting for pod/some-name-pxc-3 logs to contain 'SST retry limit reached.'+ grep -q 'SST retry limit reached.' + kubectl_bin logs some-name-pxc-3 -c pxc + echo + desc 'verify retries stop after the limit' + set +o xtrace ----------------------------------------------------------------------------------- verify retries stop after the limit ----------------------------------------------------------------------------------- + local joiner_phase ++ kubectl_bin get pod some-name-pxc-3 -o 'jsonpath={.status.phase}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.H9k13RcHwe +++ mktemp ++ local LAST_ERR=/tmp/tmp.ETxctJFh0Q ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o 'jsonpath={.status.phase}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.H9k13RcHwe ++ cat /tmp/tmp.ETxctJFh0Q ++ rm /tmp/tmp.H9k13RcHwe /tmp/tmp.ETxctJFh0Q ++ return 0 + joiner_phase=Running + local joiner_ready ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.conditions[] | select(.type == "Ready") | .status' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jcvKHUg0Sy +++ mktemp ++ local LAST_ERR=/tmp/tmp.hbnF0Mb33r ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jcvKHUg0Sy ++ cat /tmp/tmp.hbnF0Mb33r ++ rm /tmp/tmp.jcvKHUg0Sy /tmp/tmp.hbnF0Mb33r ++ return 0 + joiner_ready=False + [[ Running != \R\u\n\n\i\n\g ]] + [[ False != \F\a\l\s\e ]] + local restart_count_before ++ get_pxc_restart_count some-name-pxc-3 ++ local pod=some-name-pxc-3 ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[] | select(.name == "pxc") | .restartCount' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nQqMRSvPHb +++ mktemp ++ local LAST_ERR=/tmp/tmp.flc3t7h6EU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nQqMRSvPHb ++ cat /tmp/tmp.flc3t7h6EU ++ rm /tmp/tmp.nQqMRSvPHb /tmp/tmp.flc3t7h6EU ++ return 0 + restart_count_before=2 + sleep_with_log 120 + local d=120 + log 'sleeping for 120 seconds' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-04-30T14:30:38+0000]' sleeping for 120 seconds [2026-04-30T14:30:38+0000] sleeping for 120 seconds + sleep 120 + local restart_count_after ++ get_pxc_restart_count some-name-pxc-3 ++ local pod=some-name-pxc-3 ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[] | select(.name == "pxc") | .restartCount' +++ mktemp ++ local LAST_OUT=/tmp/tmp.O2gS3hwRIQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.yheriwCa3r ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.O2gS3hwRIQ ++ cat /tmp/tmp.yheriwCa3r ++ rm /tmp/tmp.O2gS3hwRIQ /tmp/tmp.yheriwCa3r ++ return 0 + restart_count_after=2 + [[ 2 != \2 ]] + desc 'verify donor candidates remain healthy' + set +o xtrace ----------------------------------------------------------------------------------- verify donor candidates remain healthy ----------------------------------------------------------------------------------- + local donor_result + local donor_pod + for donor_pod in '"${donor_candidates[@]}"' ++ run_mysql_local 'SELECT COUNT(*) FROM myApp.myApp;' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 ++ local 'command=SELECT COUNT(*) FROM myApp.myApp;' ++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' ++ local pod=some-name-pxc-1 ++ local container_name= ++ set +o xtrace + donor_result=1 + [[ 1 != \1 ]] + for donor_pod in '"${donor_candidates[@]}"' ++ run_mysql_local 'SELECT COUNT(*) FROM myApp.myApp;' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-2 ++ local 'command=SELECT COUNT(*) FROM myApp.myApp;' ++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' ++ local pod=some-name-pxc-2 ++ local container_name= ++ set +o xtrace + donor_result=1 + [[ 1 != \1 ]] + destroy sst-retry-limit-22407 + local namespace=sst-retry-limit-22407 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v level=info + grep -v 'the object has been modified' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ + sort -u + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + tee /tmp/tmp.X6S66n74BV/operator.log + grep -v 'get backup status: Job.batch' +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.2tTRkC9zct +++ mktemp ++ local LAST_ERR=/tmp/tmp.HwzUV0pm7N ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2tTRkC9zct ++ cat /tmp/tmp.HwzUV0pm7N ++ rm /tmp/tmp.2tTRkC9zct /tmp/tmp.HwzUV0pm7N ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-7d9c4fb94c-ghkzd ++ mktemp + local LAST_OUT=/tmp/tmp.2d8E7nRONG ++ mktemp + local LAST_ERR=/tmp/tmp.YY7b5UzDce + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-7d9c4fb94c-ghkzd + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2d8E7nRONG + cat /tmp/tmp.YY7b5UzDce + rm /tmp/tmp.2d8E7nRONG /tmp/tmp.YY7b5UzDce + return 0 2026-04-30T14:18:30.105Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.32.13-gke.1362000"} 2026-04-30T14:18:30.106Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-04-30T14:18:30.106Z INFO setup Manager starting up {"gitCommit": "b47912f7729e2d124f0221cb70713ba4110c1fe4", "gitBranch": "PR-2449-b47912f7", "buildTime": "2026-04-30T10:36:22Z", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"} 2026-04-30T14:18:30.109Z INFO setup Registering Components. 2026-04-30T14:18:30.601Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-04-30T14:18:30.601Z INFO controller-runtime.metrics Starting metrics server 2026-04-30T14:18:30.601Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-04-30T14:18:30.601Z INFO controller-runtime.webhook Starting webhook server 2026-04-30T14:18:30.601Z INFO setup Starting the Cmd. 2026-04-30T14:18:30.601Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-04-30T14:18:30.602Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-04-30T14:18:30.602Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-04-30T14:18:30.602Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-04-30T14:18:30.702Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-04-30T14:18:30.744Z DEBUG events percona-xtradb-cluster-operator-7d9c4fb94c-ghkzd_973d1c02-fc5a-4e6e-b27f-6d70532dbd60 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"489647c4-e986-4307-bf39-fecba4cf2d04","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1777558710737679009"}, "reason": "LeaderElection"} 2026-04-30T14:18:30.744Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-04-30T14:18:30.744Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-04-30T14:18:30.744Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-04-30T14:18:30.744Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-04-30T14:18:30.744Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-04-30T14:18:30.844Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-04-30T14:18:30.844Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-04-30T14:18:30.845Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-04-30T14:18:30.845Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-04-30T14:18:30.846Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-04-30T14:18:30.846Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-04-30T14:19:38.393Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "c9ca63e0-869f-4286-a17c-230b0dcc1797", "version": "1.20.0"} 2026-04-30T14:19:38.877Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "c9ca63e0-869f-4286-a17c-230b0dcc1797", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-30T14:19:38.948Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "c9ca63e0-869f-4286-a17c-230b0dcc1797", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-04-30T14:19:39.067Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "c9ca63e0-869f-4286-a17c-230b0dcc1797", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-30T14:19:39.113Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "c9ca63e0-869f-4286-a17c-230b0dcc1797", "object": "some-name-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-04-30T14:19:39.841Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "bbf85f2f-081b-4c21-a87e-326b6103072f", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-04-30T14:21:00.697Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e7925b5f-a8ec-45e6-adc5-4288102b92bf", "user": "operator"} 2026-04-30T14:21:00.740Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e7925b5f-a8ec-45e6-adc5-4288102b92bf", "user": "monitor"} 2026-04-30T14:21:00.812Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e7925b5f-a8ec-45e6-adc5-4288102b92bf"} 2026-04-30T14:21:00.854Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e7925b5f-a8ec-45e6-adc5-4288102b92bf"} 2026-04-30T14:21:00.892Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e7925b5f-a8ec-45e6-adc5-4288102b92bf", "user": "xtrabackup"} 2026-04-30T14:21:00.949Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e7925b5f-a8ec-45e6-adc5-4288102b92bf"} 2026-04-30T14:21:00.988Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e7925b5f-a8ec-45e6-adc5-4288102b92bf", "user": "replication"} 2026-04-30T14:21:00.990Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e7925b5f-a8ec-45e6-adc5-4288102b92bf", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:21:06.094Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "1c6cd0cd-5ba8-4ec6-befd-0f14b067912c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:21:11.199Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "7f3a4d2f-98bb-4223-a0f4-8f1c3850d48d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:21:16.302Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "271ca342-a968-44d9-bc65-1b8915e289b1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:21:21.408Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "2e8da18e-38c6-4062-b663-f6e72cf91550", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:21:26.507Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e857a19b-3936-4483-a11f-1b9aa59a94c6", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:21:31.631Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "400a4ee4-beae-41ac-97ab-808642a31d37", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:21:36.756Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "8cc42a9c-8bb0-424b-83b2-a1cbdf64dbba", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:21:41.852Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "0340c6ea-2008-452e-821c-81bb528a845f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:21:46.949Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e6d61724-94c6-42eb-bf67-6a3769357aa6", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:21:52.041Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "99ce2ed9-d1a3-4108-b383-175b6e549d27", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:21:57.147Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "db3fd69f-3808-4420-8831-7169318de247", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:22:02.263Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "6436ec08-c267-4330-9be0-ec488310f508", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:22:07.371Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "b5007432-56e5-4fe1-a9e4-bccc5ad0afaf", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:22:12.807Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "135223da-1169-43df-a2a6-81de40cefcf0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:22:17.980Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "6ae70943-9adb-41d0-a6cc-875c36709824", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:22:18.203Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "22df9c32-5ae5-415e-9685-2e4542b0ef99", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:22:23.355Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "95b36cb3-57c6-4b04-8526-9700b2569bfa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:22:28.493Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "eae6db4a-7531-4f10-9e26-a03c1ae4ec43", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:22:33.620Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "d43af0d8-dd9e-40b9-a3d7-63c839a0b0bb", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:22:38.764Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "36085d12-9b9a-4143-b7b6-b630f83ac2c8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:22:43.901Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "dc234ca9-6f47-4618-9b8a-60c70251efb3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:22:49.035Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "2a25c6eb-9803-4c62-be69-b2c59b30a3fe", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:22:54.191Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "be72bb18-b4ee-46f1-b299-b79f4301e9ff", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:22:59.332Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "88fa5ff2-c386-43da-9230-eb0f6cb28035", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:23:04.483Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "0e9c142f-7975-4522-9db7-80fafd5c7f25", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:23:09.624Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "0dab4e83-4d92-415f-bc82-cbd132e4e120", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:23:14.799Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e3990f1f-c8cd-4400-b8db-4f5d94cbba09", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:23:19.932Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "dda416b5-5c64-4470-82f2-d6ee14666fef", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:23:25.061Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e42f6d79-6287-4fd6-89d5-c78975ebc274", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:23:30.207Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "dcbd45f9-b3d9-4576-b9eb-b6d3e5846163", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:23:35.374Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "59dceba2-2940-40b4-ab00-5c460ce4cc0a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:23:36.133Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "06ad5799-da21-47b8-866d-4e2dd5950fc3", "user": "root"} 2026-04-30T14:23:36.203Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "06ad5799-da21-47b8-866d-4e2dd5950fc3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:23:36.211Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "06ad5799-da21-47b8-866d-4e2dd5950fc3", "new version": "8.0.43-34.1"} 2026-04-30T14:23:36.964Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "28ad675e-08d6-41bc-ad2c-df6b316cdac5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:23:42.922Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "a6cdf306-53b6-4392-b1ec-a48d0fc97af5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:23:48.163Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "5ad8e874-0358-4ed4-ac6e-c3b5ba506b6a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:23:53.379Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "12074a20-0d69-41f3-bb86-d848a3377d23", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:23:58.614Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "6443e4d7-8f49-4db6-8f2a-6cfeeb757cdd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:24:03.920Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "8e171d56-ef77-4606-9daa-434d25c2c80f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:24:09.162Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "1a2b552e-c3e6-4500-bbd9-7f3cb40c769f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:24:14.402Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "56fcb16e-198f-4a8f-9e52-d299b2d87426", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:24:19.623Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "435ad772-fa87-4d9e-b2f2-1e691845b14a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:24:24.905Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "cf5baa50-0cde-41ee-af40-ebee335fb972", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:24:30.120Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "4021c2ee-d037-4bfd-9a64-b031894c7515", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:24:35.356Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "a46d1afb-2d27-4836-af5e-438df43a41e1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:24:40.571Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "2b62789e-8b85-4970-8224-dbbcba08aee6", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:24:45.811Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "594dde20-0851-46a4-9cfe-48446fbe2c4b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:24:51.120Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "5639e793-bd76-4a65-bc8f-af2d0899ab4f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:24:56.357Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "1caf383c-d6b6-4695-9eba-4ee5b31d95eb", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:01.613Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "345e939d-7872-42ee-bc07-385ef11a8975", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:06.847Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "9976b274-802b-47a1-a48d-e70dcc94de4c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:09.035Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e96c2d26-98aa-4bf7-8caa-3e4362f031a8", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-04-30T14:25:09.124Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e96c2d26-98aa-4bf7-8caa-3e4362f031a8", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-04-30T14:25:09.187Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e96c2d26-98aa-4bf7-8caa-3e4362f031a8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:09.887Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "ab62663b-0419-4d55-8b9f-10f6870ac3d9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:15.529Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "42c198d5-eef9-4634-bb68-41643dcadfda", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:20.695Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "6df5afd9-775d-45eb-bda1-562e9cb47833", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:25.844Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "a3961c03-1094-4280-94b7-17607299a9b8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:31.034Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "c721afe9-333d-4388-b452-08cfd6b00d5e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:36.208Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "38f0c6e3-49d1-45c7-9de8-226102e3e5d7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:41.361Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "35332c7e-505a-43ed-ab68-fdcb969b3e03", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:46.519Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "1880a880-6c1e-4e16-a6e5-72e83c777d56", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:51.648Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "ad9b13d6-d9f0-4db7-9f61-b6916b615646", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:51.785Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "6f455398-9f08-45db-8306-86d03b2e9e4c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:25:56.910Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "664893d1-4f7a-495b-a51b-9df90affcfd4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:26:02.040Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "3d7882db-63e7-423e-8b96-1ae3f5701daf", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:26:07.193Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "8ff860cf-2a45-4656-afc4-15ae5b170dde", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:26:12.659Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "5e7902fe-032f-431d-9f29-b57613b9fea9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:26:18.125Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "d0b85630-d637-4693-a5d5-a14d26dd0861", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:26:23.246Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "13df6bfa-3e47-41c1-bae0-125f93591104", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:26:28.369Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "21a9737c-10d4-4a66-befd-9b5ebca2e790", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:26:33.465Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "0a442c42-ea8a-46ae-b6dc-1d8be7a93917", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:26:38.575Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "7534e4e3-3de2-45f9-abff-c50524213448", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:26:43.706Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "2eef9f33-4fb8-49d9-ab9e-ebd3051a3855", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:26:43.872Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "89d6e1ea-6d8c-46d8-ac85-53aca6f72ef5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:26:49.575Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "8e15a6e2-a46d-4720-b791-874f48c7da39", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:26:54.732Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "257c2ad9-c531-4dfd-8d7f-40ef7ad77423", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:26:59.907Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "6c4756af-dde2-4284-8289-42e25a48b968", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:27:05.062Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "f66e5911-9841-49b5-ac0e-7974925485ae", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:27:10.210Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "c4992f11-8b9e-4cc5-bf32-5ded73559bc8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:27:15.365Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "320e95bb-94e9-4414-af00-62c99426c915", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:27:20.520Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "b6fedb6b-36a7-4109-9e4f-075f6e928e18", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:27:25.683Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "f9a0f09a-21e0-4786-bf1c-d6c823674c01", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:27:30.831Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "3ea11c95-65c6-4cf1-b33f-c6cbb706c92b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:27:35.974Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "1f3d2c30-304e-4e55-bfe7-092335feb499", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:27:41.120Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "b2d10188-f88e-4b89-835e-7aab495b0f0d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:27:46.318Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "69851b08-3988-48dd-bcda-b5b9d937714c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:27:51.492Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "b8732f7e-4ec7-4944-b77d-23b5b29480a7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:27:56.646Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "520d16bb-2ed6-47f1-be8e-c88569ddab29", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:28:01.819Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "03e4a5ae-6fef-4811-bdbb-551b00837dd4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:28:06.983Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "99fb5ee9-1a3d-4d18-8a0f-8a42a9a9719e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:28:12.139Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "df6c160c-39be-4cde-8d29-2ce58c668651", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:28:17.284Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "77972019-4d76-470a-a367-d1ac99598d4c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:28:17.408Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "374ac67e-5997-415a-acf8-4c0d0710dd09", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:28:22.510Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "cc07e9ac-00dd-4b81-bce5-651aa965de0e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:28:27.757Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "fc9151d0-c7fa-431b-9558-71744341b6a5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:28:32.965Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "74c14173-5a13-4825-b33d-49c0977b6f82", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:28:38.068Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "b902fe3e-8bee-432e-bd0c-1e4dc162dbb8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:28:43.188Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "71409829-4d96-44f5-b9b9-adb36caf28e9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:28:48.297Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "88ca3f8d-c076-4343-9141-9c93d5797aa8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:28:53.393Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "a929b567-8253-46bb-b3db-4d9f41608dfb", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:28:58.498Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "72a715bd-0564-47fd-804c-bd3976ca1a8b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:29:03.602Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "57a49d3f-eebb-4f67-846a-4a23779cc4a5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:29:08.722Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "842c794c-ad95-4636-be47-32c356e8c23f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:29:08.929Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e1cd11ee-caa9-42db-9487-40531067a34d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:29:14.570Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "2a0dd0ed-e939-4329-b435-a5a24d21c99c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:29:19.726Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "02fb2aa0-fc75-4ba9-8505-22fbe62ad67a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:29:24.895Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "daf2b0f6-b10a-4ffa-a10d-daca7910d2be", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:29:30.036Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "a70d2a31-3456-46c1-afbd-3db5e5ac33c3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:29:35.191Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "82068f82-f93f-4536-852a-e328471f2e38", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:29:40.342Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "8b9109d5-f195-494c-9dcc-5ac7a2df115b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:29:45.490Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "dcfd4c6a-2a42-4af2-9428-35e4ffaf0311", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:29:50.664Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "ef548986-d152-4a6c-b80d-335f4bfc6220", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:29:55.834Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "65479b55-5b6b-4e0f-9558-7f6fb0708602", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:00.985Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "eda7d614-f331-4cb2-99ba-5b143b046e76", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:06.166Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "d9dfe268-a355-46cb-94b4-61f22002b4b2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:11.335Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "fc72a99d-a3cb-4d1d-8a18-ea36de342f3a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:16.498Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "a02d4455-15aa-4686-8071-36fc38b3cf2f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:16.679Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "0066330f-51ff-44ca-815d-493b54142fc0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:22.355Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "7b85aa19-e24b-40b1-aa09-3b68ee84fea3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:27.832Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "42ef462c-b57a-4504-8bd6-addc782ee81c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:32.988Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "55a38630-ff28-4adc-9066-e124ef18d633", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:33.165Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "c6466471-c16b-46ae-b244-2e448209cafe", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:38.839Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "de3148b2-c3af-4915-9298-b4a7f468e76b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:44.018Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "130c7565-f798-4db6-accd-551e095256a2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:49.181Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "a436009f-712b-4adb-8335-008c15676cf3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:54.348Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "c80c5357-55ba-4a89-97fe-79e649549cbe", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:30:59.517Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "98602202-39c2-472e-834e-97eaec3a5359", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:31:04.737Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "f97e3c03-39ba-49d5-9637-7decd058e9ee", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:31:09.936Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "8c93cf35-ee11-4735-ac96-fe7b9df6adfa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:31:15.131Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "1f3aab05-0900-4635-875a-36ff58bb5ce1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:31:20.295Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "a0864985-9eb1-4a35-b816-f7c8eae3ed18", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:31:25.507Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "60368215-cef9-4a48-a631-bf766f94f354", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:31:30.789Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "b3ef11a0-b230-4638-95df-c7f9105040b4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:31:35.957Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "b1d740d1-9ab6-4a1f-9ded-35cc43f0f92e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:31:41.139Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "2481a4fb-7652-4bdb-a09e-381bcbc1db7e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:31:46.304Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "b2d2529e-0608-4db4-94cf-7825aa6a2168", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:31:51.504Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "8ce7ffeb-068e-461d-a7c8-d6fa13f04941", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:31:56.660Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "716ff767-e5b2-4633-9a0a-a6e285f72f30", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:32:01.849Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "d71895e0-b719-438a-9238-e28bb21a386e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:32:07.012Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e46f69c9-c730-49b5-b9b4-7e7c9cf9a9ec", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:32:12.167Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "363bb449-e5ea-44ea-98d0-507abf0d8938", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:32:17.610Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "fe24bbba-c9b4-4d0f-af21-8a7b0379cf84", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:32:22.761Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "e1ebb462-4d65-4993-9c6f-6cf5c2e64916", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:32:27.934Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "2f9bf28b-3efd-40c7-8e3f-81587baf0483", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:32:33.124Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "33dda3b4-a86d-47a4-88e8-9b03ee662229", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-04-30T14:32:38.279Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22407"}, "namespace": "sst-retry-limit-22407", "name": "some-name", "reconcileID": "2b686906-8091-4e29-a394-f2f6b8cf1f1d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} -  }, -  { -  }, +  }, -  Annotations: map[string]string{ +  Annotations: map[string]string{ -  APIVersion: "apps/v1", -  APIVersion: "apps/v1", +  AvailableReplicas: 0, -  AvailableReplicas: 3, -  CollisionCount: &0, +  CollisionCount: nil, +  CreationTimestamp: v1.Time{}, -  CreationTimestamp: v1.Time{Time: s"2026-04-30 14:19:38 +0000 UTC"}, +  CurrentReplicas: 0, -  CurrentReplicas: 3, +  CurrentRevision: "", -  CurrentRevision: "some-name-pxc-7978846788", -  DefaultMode: &420, -  DefaultMode: &420, +  DefaultMode: nil, +  DefaultMode: nil, +  DeprecatedServiceAccount: "", -  DeprecatedServiceAccount: "default", +  DNSPolicy: "", -  DNSPolicy: "ClusterFirst", -  FieldsType: "FieldsV1", -  FieldsType: "FieldsV1", -  FieldsV1: s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`..., -  FieldsV1: s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`..., +  Generation: 0, -  Generation: 1, +  ManagedFields: nil, -  ManagedFields: []v1.ManagedFieldsEntry{ -  Manager: "kube-controller-manager", -  Manager: "percona-xtradb-cluster-operator", +  ObservedGeneration: 0, -  ObservedGeneration: 1, -  Operation: "Update", -  Operation: "Update", -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJl"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6NCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJl"..., +  PeriodSeconds: 0, -  PeriodSeconds: 10, +  PersistentVolumeClaimRetentionPolicy: nil, -  PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}", +  Phase: "", -  Phase: "Pending", +  PodManagementPolicy: "", -  PodManagementPolicy: "OrderedReady", +  Protocol: "", -  Protocol: "TCP", +  ReadyReplicas: 0, -  ReadyReplicas: 3, +  Replicas: 0, -  Replicas: 3, -  Replicas: &3, +  Replicas: &4, +  ResourceVersion: "", -  ResourceVersion: "1777559014008575010", +  RestartPolicy: "", -  RestartPolicy: "Always", -  RevisionHistoryLimit: &10, +  RevisionHistoryLimit: nil, +  SchedulerName: "", -  SchedulerName: "default-scheduler", -  Subresource: "status", +  TerminationMessagePath: "", -  TerminationMessagePath: "/dev/termination-log", +  TerminationMessagePolicy: "", -  TerminationMessagePolicy: "File", -  Time: s"2026-04-30 14:19:38 +0000 UTC", -  Time: s"2026-04-30 14:23:33 +0000 UTC", -  TopologySpreadConstraints: nil, +  TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, +  UID: "", -  UID: "586298af-043c-4434-8618-226656da79ac", +  UpdatedReplicas: 0, -  UpdatedReplicas: 3, +  UpdateRevision: "", -  UpdateRevision: "some-name-pxc-7978846788", -  VolumeMode: &"Filesystem", +  VolumeMode: nil,   }    },    },    {    },    },    {    },    },    {    },    ... // 16 identical fields    ... // 16 identical fields    ... // 22 identical fields    ... // 2 identical fields    ... // 3 identical fields    ... // 3 identical fields    ... // 3 identical fields    ... // 4 identical fields    ... // 5 identical fields    ... // 6 identical fields    ... // 7 identical fields    ... // 9 identical fields    AccessModes: nil,    ActiveDeadlineSeconds: nil,    Affinity: nil,    Args: {"mysqld"},    AutomountServiceAccountToken: nil,    AWSElasticBlockStore: nil,    AzureFile: nil,    Capacity: nil,    Conditions: nil,    ConfigMap: &v1.ConfigMapVolumeSource{    ContainerPort: 3306,    ContainerPort: 33060,    ContainerPort: 33062,    ContainerPort: 4444,    ContainerPort: 4567,    ContainerPort: 4568,    Containers: []v1.Container{    DataSource: nil,    DataSourceRef: nil,    DeletionGracePeriodSeconds: nil,    DeletionTimestamp: nil,    EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "some-name-env-vars-pxc"}, Optional: &true}}},    Env: {{Name: "PXC_SERVICE", Value: "some-name-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name"}, Key: "xtrabackup"}}}, ...},    EphemeralContainers: nil,    FailureThreshold: 3,    FC: nil,    Finalizers: nil,    GitRepo: nil,    HostAliases: nil,    HostIP: "",    HostPort: 0,    ImagePullPolicy: "IfNotPresent",    InitContainers: []v1.Container{    InitialDelaySeconds: 300,    ISCSI: nil,    Items: nil,    Items: nil,    Labels: nil,    Lifecycle: nil,    LivenessProbe: &v1.Probe{    LocalObjectReference: {Name: "auto-some-name-pxc"},    LocalObjectReference: {Name: "some-name-pxc"},    MinReadySeconds: 0,    Name: "auto-config",    {Name: "bin", VolumeSource: {EmptyDir: &{}}},    Name: "config",    Name: "ist",    Name: "mysql",    Name: "mysql-admin",    Name: "mysql-init-file",    Name: "mysql-users-secret-file",    Name: "mysqlx",    Namespace: "sst-retry-limit-22407",    Name: "ssl",    Name: "ssl-internal",    Name: "sst",    {Name: "tmp", VolumeSource: {EmptyDir: &{}}},    Name: "vault-keyring-secret",    Name: "write-set",    NFS: nil,    NodeName: "",    NodeSelector: nil,    ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "cfd6a52398268173b51d7cdb1331c09a", "percona.com/ssl-internal-hash": "751e59a1c1d94ac67f9d6bb6e756acdc"}},    ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    ObjectMeta: v1.ObjectMeta{    Optional: &false,    Optional: &true,    Optional: &true,    Ordinals: nil,    OS: nil,    Overhead: nil,    OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "some-name", UID: "748a75c6-9547-4491-9cdb-45ea534b53f7", ...}},    Ports: []v1.ContainerPort{    PreemptionPolicy: nil,    ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}},    Quobyte: nil,    ReadinessProbe: &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...},    SecretName: "internal-some-name",    SecretName: "some-name-mysql-init",    SecretName: "some-name-ssl",    SecretName: "some-name-ssl-internal",    SecretName: "some-name-vault",    Secret: &v1.SecretVolumeSource{    SecurityContext: nil,    Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    SelfLink: "",    ServiceAccountName: "default",    ServiceName: "some-name-pxc",    SetHostnameAsFQDN: nil,    Spec: v1.PersistentVolumeClaimSpec{    Spec: v1.PodSpec{    Spec: v1.StatefulSetSpec{    StartupProbe: nil,    Status: v1.PersistentVolumeClaimStatus{    Status: v1.StatefulSetStatus{    StorageClassName: nil,    Subdomain: "",    SuccessThreshold: 1,    Template: v1.PodTemplateSpec{    TerminationGracePeriodSeconds: &600,    TerminationGracePeriodSeconds: nil,    TimeoutSeconds: 5,    Tolerations: nil,    TypeMeta: {},    TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"},    UpdateStrategy: {Type: "RollingUpdate", RollingUpdate: &{Partition: &0}},   &v1.StatefulSet{    VolumeAttributesClassName: nil,    VolumeClaimTemplates: []v1.PersistentVolumeClaim{    VolumeDevices: nil,    VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...},    VolumeName: "",    VolumeSource: v1.VolumeSource{    Volumes: []v1.Volume{    VsphereVolume: nil,    WorkingDir: "", + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl patch pxc -n sst-retry-limit-22407 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.yXGSUG9Sfb ++ mktemp + local LAST_ERR=/tmp/tmp.aOxaJ2URfW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yXGSUG9Sfb perconaxtradbcluster.pxc.percona.com "some-name" deleted from sst-retry-limit-22407 namespace + cat /tmp/tmp.aOxaJ2URfW + rm /tmp/tmp.yXGSUG9Sfb /tmp/tmp.aOxaJ2URfW + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.551dS5Khfa ++ mktemp + local LAST_ERR=/tmp/tmp.Q8piPM9HMG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.551dS5Khfa No resources found + cat /tmp/tmp.Q8piPM9HMG + rm /tmp/tmp.551dS5Khfa /tmp/tmp.Q8piPM9HMG + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.MxtM5nfbNe ++ mktemp + local LAST_ERR=/tmp/tmp.p2P5r8APTg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MxtM5nfbNe No resources found + cat /tmp/tmp.p2P5r8APTg + rm /tmp/tmp.MxtM5nfbNe /tmp/tmp.p2P5r8APTg + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.clwAmPiWed ++ mktemp + local LAST_ERR=/tmp/tmp.5ySwla65qd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.clwAmPiWed validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.5ySwla65qd + rm /tmp/tmp.clwAmPiWed /tmp/tmp.5ySwla65qd + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace sst-retry-limit-22407 + rm -rf /tmp/tmp.X6S66n74BV + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ mktemp + desc 'test passed' + set +o xtrace + local LAST_OUT=/tmp/tmp.UaX0ErPMg6 ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.WGAOp0qcxR ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.qxZhzBRaxU + local exit_status=0 + local LAST_ERR=/tmp/tmp.pA6OzRqROC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace sst-retry-limit-22407 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator