Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/logs/sst-retry-limit-8-0.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + main + local cluster=some-name + local joiner_pod=some-name-pxc-3 + donor_candidates=("${cluster}-pxc-1" "${cluster}-pxc-2") + local donor_candidates + create_infra sst-retry-limit-3087 + local ns=sst-retry-limit-3087 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n sst-retry-limit-30687 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.OiWev3KZvQ ++ mktemp + local LAST_ERR=/tmp/tmp.LgQAexUStF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OiWev3KZvQ perconaxtradbcluster.pxc.percona.com "some-name" deleted from sst-retry-limit-30687 namespace + cat /tmp/tmp.LgQAexUStF + rm /tmp/tmp.OiWev3KZvQ /tmp/tmp.LgQAexUStF + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.UgLg3FPju9 ++ mktemp + local LAST_ERR=/tmp/tmp.yy3GxATSZt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UgLg3FPju9 No resources found + cat /tmp/tmp.yy3GxATSZt + rm /tmp/tmp.UgLg3FPju9 /tmp/tmp.yy3GxATSZt + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.0OJZsDwLJa ++ mktemp + local LAST_ERR=/tmp/tmp.cPJfFXapUZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0OJZsDwLJa No resources found + cat /tmp/tmp.cPJfFXapUZ + rm /tmp/tmp.0OJZsDwLJa /tmp/tmp.cPJfFXapUZ + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl api-resources ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + kubectl_bin get ns ++ mktemp + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.gsk1cRnpBv ++ mktemp + local LAST_OUT=/tmp/tmp.8KdUgvxDeU + local LAST_ERR=/tmp/tmp.D7gGX5hlBU + local exit_status=0 ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.bkuJJktBfh + local exit_status=0 + for i in '$(seq 0 2)' + set +e + kubectl get ns ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gsk1cRnpBv + cat /tmp/tmp.D7gGX5hlBU + rm /tmp/tmp.gsk1cRnpBv /tmp/tmp.D7gGX5hlBU + return 0 namespace "sst-retry-limit-30687" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8KdUgvxDeU namespace "pxc-operator" deleted + cat /tmp/tmp.bkuJJktBfh + rm /tmp/tmp.8KdUgvxDeU /tmp/tmp.bkuJJktBfh + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.OAoRAE0za1 ++ mktemp + local LAST_ERR=/tmp/tmp.8KGHwQET2k + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OAoRAE0za1 namespace/pxc-operator created + cat /tmp/tmp.8KGHwQET2k + rm /tmp/tmp.OAoRAE0za1 /tmp/tmp.8KGHwQET2k + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.RDy88id0Ku +++ mktemp ++ local LAST_ERR=/tmp/tmp.F2FEHulbhq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RDy88id0Ku ++ cat /tmp/tmp.F2FEHulbhq ++ rm /tmp/tmp.RDy88id0Ku /tmp/tmp.F2FEHulbhq ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2473-6d392bea-4-cluster2 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.n5KhWr9mGd ++ mktemp + local LAST_ERR=/tmp/tmp.dl36ECaw3r + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2473-6d392bea-4-cluster2 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.n5KhWr9mGd Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2473-6d392bea-4-cluster2" modified. + cat /tmp/tmp.dl36ECaw3r + rm /tmp/tmp.n5KhWr9mGd /tmp/tmp.dl36ECaw3r + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.8sZ0NniEci ++ mktemp + local LAST_ERR=/tmp/tmp.Fl0u3rccLE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8sZ0NniEci customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.Fl0u3rccLE + rm /tmp/tmp.8sZ0NniEci /tmp/tmp.Fl0u3rccLE + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/deploy/cw-rbac.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.pYSBiw1BdL ++ mktemp + local LAST_ERR=/tmp/tmp.EXBaKG1Soe + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pYSBiw1BdL clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.EXBaKG1Soe + rm /tmp/tmp.pYSBiw1BdL /tmp/tmp.EXBaKG1Soe + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/deploy/cw-operator.yaml + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2473-6d392bea^' ++ mktemp + local LAST_OUT=/tmp/tmp.Hlb4eaQxTQ ++ mktemp + local LAST_ERR=/tmp/tmp.Yc9dDqSFTx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Hlb4eaQxTQ deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.Yc9dDqSFTx + rm /tmp/tmp.Hlb4eaQxTQ /tmp/tmp.Yc9dDqSFTx + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.BaIdWEvlEh ++ mktemp + local LAST_ERR=/tmp/tmp.GFb2PlDTGb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BaIdWEvlEh pod/percona-xtradb-cluster-operator-55d95dc9d8-gmfr6 condition met + cat /tmp/tmp.GFb2PlDTGb E0517 01:52:36.683462 14627 reflector.go:227] "Failed to watch" err="Get \"https://34.61.233.25/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-55d95dc9d8-gmfr6&resourceVersion=1778982756323269000&timeoutSeconds=321&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.BaIdWEvlEh /tmp/tmp.GFb2PlDTGb + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.LGharBi3Om +++ mktemp ++ local LAST_ERR=/tmp/tmp.ePzxhwQAho ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LGharBi3Om ++ cat /tmp/tmp.ePzxhwQAho ++ rm /tmp/tmp.LGharBi3Om /tmp/tmp.ePzxhwQAho ++ return 0 + wait_pod percona-xtradb-cluster-operator-55d95dc9d8-gmfr6 480 pxc-operator + local pod=percona-xtradb-cluster-operator-55d95dc9d8-gmfr6 + local max_retry=480 + local ns=pxc-operator ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo percona-xtradb-cluster-operator-55d95dc9d8-gmfr6 ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-55d95dc9d8-gmfr6 condition met E0517 01:52:43.137012 15466 reflector.go:227] "Failed to watch" err="Get \"https://34.61.233.25/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-55d95dc9d8-gmfr6&resourceVersion=1778982760466823000&timeoutSeconds=311&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-55d95dc9d8-gmfr6 to become Ready.Ok + sleep 3 + create_namespace sst-retry-limit-3087 + local namespace=sst-retry-limit-3087 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces sst-retry-limit-3087' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces sst-retry-limit-3087 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace sst-retry-limit-3087 + awk '{print$1}' ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.Fftbxq533s ++ mktemp + local LAST_OUT=/tmp/tmp.RNjMplNbBa ++ mktemp + local LAST_ERR=/tmp/tmp.zUKzUtQ4sm + local exit_status=0 + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.Qkc1XsG35P + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace sst-retry-limit-3087 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace sst-retry-limit-3087 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Fftbxq533s + cat /tmp/tmp.zUKzUtQ4sm + rm /tmp/tmp.Fftbxq533s /tmp/tmp.zUKzUtQ4sm + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace sst-retry-limit-3087 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.RNjMplNbBa + cat /tmp/tmp.Qkc1XsG35P Error from server (NotFound): namespaces "sst-retry-limit-3087" not found + rm /tmp/tmp.RNjMplNbBa /tmp/tmp.Qkc1XsG35P + return 1 + : + wait_for_delete namespace/sst-retry-limit-3087 + local res=namespace/sst-retry-limit-3087 + echo -n 'waiting for namespace/sst-retry-limit-3087 to be deleted' waiting for namespace/sst-retry-limit-3087 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "sst-retry-limit-3087" not found + desc 'create namespace sst-retry-limit-3087' + set +o xtrace ----------------------------------------------------------------------------------- create namespace sst-retry-limit-3087 ----------------------------------------------------------------------------------- + kubectl_bin create namespace sst-retry-limit-3087 ++ mktemp + local LAST_OUT=/tmp/tmp.SlJddZtcDr ++ mktemp + local LAST_ERR=/tmp/tmp.lk8gMbGzcE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace sst-retry-limit-3087 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SlJddZtcDr namespace/sst-retry-limit-3087 created + cat /tmp/tmp.lk8gMbGzcE + rm /tmp/tmp.SlJddZtcDr /tmp/tmp.lk8gMbGzcE + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.AGOHRrUqrL +++ mktemp ++ local LAST_ERR=/tmp/tmp.BfbJhk878q ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AGOHRrUqrL ++ cat /tmp/tmp.BfbJhk878q ++ rm /tmp/tmp.AGOHRrUqrL /tmp/tmp.BfbJhk878q ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2473-6d392bea-4-cluster2 --namespace=sst-retry-limit-3087 ++ mktemp + local LAST_OUT=/tmp/tmp.Blefun1lQn ++ mktemp + local LAST_ERR=/tmp/tmp.nPaYnuNjYw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2473-6d392bea-4-cluster2 --namespace=sst-retry-limit-3087 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Blefun1lQn Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2473-6d392bea-4-cluster2" modified. + cat /tmp/tmp.nPaYnuNjYw + rm /tmp/tmp.Blefun1lQn /tmp/tmp.nPaYnuNjYw + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.LD4KYJDQoD ++ mktemp + local LAST_ERR=/tmp/tmp.LsmTHhiPZV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LD4KYJDQoD secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.LsmTHhiPZV + rm /tmp/tmp.LD4KYJDQoD /tmp/tmp.LsmTHhiPZV + return 0 + desc 'start 3-node cluster with SST retry limit' + set +o xtrace ----------------------------------------------------------------------------------- start 3-node cluster with SST retry limit ----------------------------------------------------------------------------------- + spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/conf/some-name.yml 3 10 /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/secrets.yml /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/client.yml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/conf/some-name.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.IGshEDVr5j ++ mktemp + local LAST_ERR=/tmp/tmp.elzPQxUjrk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IGshEDVr5j secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.elzPQxUjrk + rm /tmp/tmp.IGshEDVr5j /tmp/tmp.elzPQxUjrk + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/client.yml + local pvc_name= + kubectl_bin apply -f - + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.sst-retry-limit-3087~ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2473-6d392bea#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.KAI8INpTds + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + local LAST_ERR=/tmp/tmp.Bsqje3PHMu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KAI8INpTds deployment.apps/pxc-client created + cat /tmp/tmp.Bsqje3PHMu + rm /tmp/tmp.KAI8INpTds /tmp/tmp.Bsqje3PHMu + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/conf/some-name.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/conf/some-name.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/conf/some-name.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/conf/some-name.yml + local pvc_name= + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2473-6d392bea#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.sst-retry-limit-3087~ ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/conf/some-name.yml + local LAST_OUT=/tmp/tmp.TUxKrGdiOT ++ mktemp + local LAST_ERR=/tmp/tmp.1KsLeM6xuM + local exit_status=0 + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TUxKrGdiOT perconaxtradbcluster.pxc.percona.com/some-name created + cat /tmp/tmp.1KsLeM6xuM + rm /tmp/tmp.TUxKrGdiOT /tmp/tmp.1KsLeM6xuM + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2vz05RImkw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0xUpzkgbWA +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.2vz05RImkw +++ cat /tmp/tmp.0xUpzkgbWA +++ rm /tmp/tmp.2vz05RImkw /tmp/tmp.0xUpzkgbWA +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Q2G8ijtR41 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.u3VBkb5uvO +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Q2G8ijtR41 +++ cat /tmp/tmp.u3VBkb5uvO +++ rm /tmp/tmp.Q2G8ijtR41 /tmp/tmp.u3VBkb5uvO +++ return 0 ++ [[ false == \t\r\u\e ]] ++ echo some-name-pxc + local proxy=some-name-pxc + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-3087 ++ mktemp + local LAST_OUT=/tmp/tmp.KRndQTPSNt ++ mktemp + local LAST_ERR=/tmp/tmp.8zsgYfG6xQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-3087 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-3087 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-3087 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.KRndQTPSNt + cat /tmp/tmp.8zsgYfG6xQ error: no matching resources found + rm /tmp/tmp.KRndQTPSNt /tmp/tmp.8zsgYfG6xQ + return 1 + true + wait_for_running some-name-pxc 1 + local name=some-name-pxc + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ echo some-name-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met waiting for pod/some-name-pxc-0 to become Ready.Ok + wait_for_running some-name-pxc 3 + local name=some-name-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ echo some-name-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met E0517 01:54:50.145201 389 reflector.go:227] "Failed to watch" err="Get \"https://34.61.233.25/api/v1/namespaces/sst-retry-limit-3087/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-pxc-0&resourceVersion=1778982888934319000&timeoutSeconds=385&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/some-name-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-1 480 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ echo some-name-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-2 480 + local pod=some-name-pxc-2 + local max_retry=480 + local ns= ++ echo some-name-pxc-2 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-pxc-2 condition met waiting for pod/some-name-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.F3J1gnXSyD +++ mktemp ++ local LAST_ERR=/tmp/tmp.VpPyF6OTY0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.F3J1gnXSyD ++ cat /tmp/tmp.VpPyF6OTY0 ++ rm /tmp/tmp.F3J1gnXSyD /tmp/tmp.VpPyF6OTY0 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TNqekZmYxo +++ mktemp ++ local LAST_ERR=/tmp/tmp.7BIPBoF5gX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TNqekZmYxo ++ cat /tmp/tmp.7BIPBoF5gX ++ rm /tmp/tmp.TNqekZmYxo /tmp/tmp.7BIPBoF5gX ++ return 0 + client_pod=pxc-client-67fc4995bb-hw2dd + wait_pod pxc-client-67fc4995bb-hw2dd + local pod=pxc-client-67fc4995bb-hw2dd + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-hw2dd ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-hw2dd condition met E0517 01:57:47.201581 23096 reflector.go:227] "Failed to watch" err="Get \"https://34.61.233.25/api/v1/namespaces/sst-retry-limit-3087/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-hw2dd&resourceVersion=1778983066298196000&timeoutSeconds=325&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-hw2dd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ierkyvoz2s +++ mktemp ++ local LAST_ERR=/tmp/tmp.RMWgigm3dh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ierkyvoz2s ++ cat /tmp/tmp.RMWgigm3dh ++ rm /tmp/tmp.Ierkyvoz2s /tmp/tmp.RMWgigm3dh ++ return 0 + client_pod=pxc-client-67fc4995bb-hw2dd + wait_pod pxc-client-67fc4995bb-hw2dd + local pod=pxc-client-67fc4995bb-hw2dd + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-67fc4995bb-hw2dd ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-hw2dd condition met E0517 01:57:58.506039 24535 reflector.go:227] "Failed to watch" err="Get \"https://34.61.233.25/api/v1/namespaces/sst-retry-limit-3087/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-hw2dd&resourceVersion=1778983076181687000&timeoutSeconds=401&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-hw2dd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.133297EQiJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.5bSn6vR1QU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.133297EQiJ ++ cat /tmp/tmp.5bSn6vR1QU ++ rm /tmp/tmp.133297EQiJ /tmp/tmp.5bSn6vR1QU ++ return 0 + client_pod=pxc-client-67fc4995bb-hw2dd + wait_pod pxc-client-67fc4995bb-hw2dd + local pod=pxc-client-67fc4995bb-hw2dd + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-hw2dd ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-hw2dd condition met E0517 01:58:39.046937 30231 reflector.go:227] "Failed to watch" err="Get \"https://34.61.233.25/api/v1/namespaces/sst-retry-limit-3087/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-hw2dd&resourceVersion=1778983117266265000&timeoutSeconds=376&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-hw2dd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.osnukoNZUe/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.osnukoNZUe/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.osnukoNZUe/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DSvIq2QYrA +++ mktemp ++ local LAST_ERR=/tmp/tmp.96QDzm1ceV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DSvIq2QYrA ++ cat /tmp/tmp.96QDzm1ceV ++ rm /tmp/tmp.DSvIq2QYrA /tmp/tmp.96QDzm1ceV ++ return 0 + client_pod=pxc-client-67fc4995bb-hw2dd + wait_pod pxc-client-67fc4995bb-hw2dd + local pod=pxc-client-67fc4995bb-hw2dd + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-hw2dd + local container= + set +o xtrace pod/pxc-client-67fc4995bb-hw2dd condition met E0517 01:58:51.685158 32019 reflector.go:227] "Failed to watch" err="Get \"https://34.61.233.25/api/v1/namespaces/sst-retry-limit-3087/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-hw2dd&resourceVersion=1778983129730665000&timeoutSeconds=492&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-hw2dd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.osnukoNZUe/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.osnukoNZUe/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.osnukoNZUe/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.m6bsl5eUj0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.DBwWLbC1sp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.m6bsl5eUj0 ++ cat /tmp/tmp.DBwWLbC1sp ++ rm /tmp/tmp.m6bsl5eUj0 /tmp/tmp.DBwWLbC1sp ++ return 0 + client_pod=pxc-client-67fc4995bb-hw2dd + wait_pod pxc-client-67fc4995bb-hw2dd + local pod=pxc-client-67fc4995bb-hw2dd + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-hw2dd ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-hw2dd condition met E0517 01:59:04.660516 1648 reflector.go:227] "Failed to watch" err="Get \"https://34.61.233.25/api/v1/namespaces/sst-retry-limit-3087/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-hw2dd&resourceVersion=1778983143694169000&timeoutSeconds=373&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-hw2dd to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.osnukoNZUe/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.osnukoNZUe/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2473/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.osnukoNZUe/select-1.sql + is_keyring_plugin_in_use some-name + local cluster=some-name + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + grep -E -o 'early-plugin-load=keyring_\w+.so' + kubectl exec some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + return 1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.secretsName}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4puyQDMEdR +++ mktemp ++ local LAST_ERR=/tmp/tmp.3ZK1h4ZRF4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4puyQDMEdR ++ cat /tmp/tmp.3ZK1h4ZRF4 ++ rm /tmp/tmp.4puyQDMEdR /tmp/tmp.3ZK1h4ZRF4 ++ return 0 + secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.Nqh2xz8Sto +++ mktemp ++ local LAST_ERR=/tmp/tmp.YVECCcQeSL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Nqh2xz8Sto ++ cat /tmp/tmp.YVECCcQeSL ++ rm /tmp/tmp.Nqh2xz8Sto /tmp/tmp.YVECCcQeSL ++ return 0 + root_pass=root_password + desc 'scale to 4 nodes to trigger SST' + set +o xtrace ----------------------------------------------------------------------------------- scale to 4 nodes to trigger SST ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type=merge -p '{"spec":{"pxc":{"size":4}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.JrPmFdV9WJ ++ mktemp + local LAST_ERR=/tmp/tmp.BL4q7zynRi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc some-name --type=merge -p '{"spec":{"pxc":{"size":4}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JrPmFdV9WJ perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.BL4q7zynRi + rm /tmp/tmp.JrPmFdV9WJ /tmp/tmp.BL4q7zynRi + return 0 + desc 'verify the joiner pod has SST retry support wired in' + set +o xtrace ----------------------------------------------------------------------------------- verify the joiner pod has SST retry support wired in ----------------------------------------------------------------------------------- + local retry + retry=0 + echo -n 'waiting for pod/some-name-pxc-3 container/pxc to start' waiting for pod/some-name-pxc-3 container/pxc to start+ local joiner_container_started ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.lALGH3vuQZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.7dp4m72Kee ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lALGH3vuQZ ++ cat /tmp/tmp.7dp4m72Kee ++ rm /tmp/tmp.lALGH3vuQZ /tmp/tmp.7dp4m72Kee ++ return 0 + joiner_container_started= + [[ '' == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.bQyPjSHySU +++ mktemp ++ local LAST_ERR=/tmp/tmp.iqdiaF1ODI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bQyPjSHySU ++ cat /tmp/tmp.iqdiaF1ODI ++ rm /tmp/tmp.bQyPjSHySU /tmp/tmp.iqdiaF1ODI ++ return 0 + joiner_container_started= + [[ '' == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zEWa3nHBCX +++ mktemp ++ local LAST_ERR=/tmp/tmp.GgYxfAm00E ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zEWa3nHBCX ++ cat /tmp/tmp.GgYxfAm00E ++ rm /tmp/tmp.zEWa3nHBCX /tmp/tmp.GgYxfAm00E ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 240 ]] ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.q7TaTikmAr +++ mktemp ++ local LAST_ERR=/tmp/tmp.kCNHUTElZ6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.q7TaTikmAr ++ cat /tmp/tmp.kCNHUTElZ6 ++ rm /tmp/tmp.q7TaTikmAr /tmp/tmp.kCNHUTElZ6 ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 240 ]] ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.GkNbYxzd7Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.KFWbE1uyUD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GkNbYxzd7Z ++ cat /tmp/tmp.KFWbE1uyUD ++ rm /tmp/tmp.GkNbYxzd7Z /tmp/tmp.KFWbE1uyUD ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Je0p0YLE4W +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ft9xsmL4f7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Je0p0YLE4W ++ cat /tmp/tmp.Ft9xsmL4f7 ++ rm /tmp/tmp.Je0p0YLE4W /tmp/tmp.Ft9xsmL4f7 ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.66L5QFLbNS +++ mktemp ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' ++ local LAST_ERR=/tmp/tmp.snGDgKNrvE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.66L5QFLbNS ++ cat /tmp/tmp.snGDgKNrvE ++ rm /tmp/tmp.66L5QFLbNS /tmp/tmp.snGDgKNrvE ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=7 + [[ 7 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iaOQUTzkg8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.sFyYNUAMfQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iaOQUTzkg8 ++ cat /tmp/tmp.sFyYNUAMfQ ++ rm /tmp/tmp.iaOQUTzkg8 /tmp/tmp.sFyYNUAMfQ ++ return 0 + joiner_container_started=true + [[ true == \t\r\u\e ]] + echo + desc 'interrupt SST twice so the joiner exhausts allowed retries' + set +o xtrace ----------------------------------------------------------------------------------- interrupt SST twice so the joiner exhausts allowed retries ----------------------------------------------------------------------------------- + interrupt_donor_during_sst root_password some-name-pxc-3 0 some-name-pxc-1 some-name-pxc-2 + local root_pass=root_password + local joiner_pod=some-name-pxc-3 + local current_restart_count=0 + local target_restart_count=1 + local donor_pod + donor_candidates=("${@:4}") + local donor_candidates + local retry=0 + local observed_restart_count + echo -n 'waiting for pod/some-name-pxc-3 restartCount=0 SST attempt' waiting for pod/some-name-pxc-3 restartCount=0 SST attempt+ : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=0 + [[ 0 == \0 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + grep -q 'Waiting for SST streaming to complete!' + kubectl_bin logs some-name-pxc-3 -c pxc + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] + : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=0 + [[ 0 == \0 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + echo + break ++ detect_active_donor root_password 120 some-name-pxc-1 some-name-pxc-2 ++ local root_pass=root_password ++ shift ++ local max_retry=120 ++ shift ++ local retry=0 ++ local donor_pod ++ local donor_rejects_queries ++ local local_state ++ echo -n 'waiting to detect active donor' waiting to detect active donor++ : ++ for donor_pod in '"$@"' +++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}' +++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace ++ donor_rejects_queries=OFF +++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace +++ awk '$1 == "wsrep_local_state_comment" {print $2}' ++ local_state=Donor/Desynced ++ [[ OFF == \O\N ]] ++ [[ Donor/Desynced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]] ++ echo some-name-pxc-1 ++ return 0 + donor_pod=some-name-pxc-1 + echo 'interrupting donor pod/some-name-pxc-1 during SST attempt 1' interrupting donor pod/some-name-pxc-1 during SST attempt 1 + kubectl_bin delete pod some-name-pxc-1 --wait=false --grace-period=0 --force ++ mktemp + local LAST_OUT=/tmp/tmp.utVQGNwEHq ++ mktemp + local LAST_ERR=/tmp/tmp.kr4lsgMvzq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod some-name-pxc-1 --wait=false --grace-period=0 --force + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.utVQGNwEHq pod "some-name-pxc-1" force deleted from sst-retry-limit-3087 namespace + cat /tmp/tmp.kr4lsgMvzq Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.utVQGNwEHq /tmp/tmp.kr4lsgMvzq + return 0 + wait_for_pxc_restart_count some-name-pxc-3 1 600 + local pod=some-name-pxc-3 + local expected=1 + local max_retry=600 + local retry=0 + echo -n 'waiting for pod/some-name-pxc-3 restartCount to reach 1' waiting for pod/some-name-pxc-3 restartCount to reach 1++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=7 + [[ 7 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=8 + [[ 8 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=9 + [[ 9 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=10 + [[ 10 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=11 + [[ 11 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=12 + [[ 12 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=13 + [[ 13 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=14 + [[ 14 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=15 + [[ 15 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=16 + [[ 16 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=17 + [[ 17 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=18 + [[ 18 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=19 + [[ 19 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=20 + [[ 20 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=21 + [[ 21 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=22 + [[ 22 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=23 + [[ 23 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=24 + [[ 24 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=25 + [[ 25 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=26 + [[ 26 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=27 + [[ 27 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=28 + [[ 28 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=29 + [[ 29 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=30 + [[ 30 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=31 + [[ 31 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=32 + [[ 32 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=33 + [[ 33 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=34 + [[ 34 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=35 + [[ 35 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=36 + [[ 36 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=37 + [[ 37 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=38 + [[ 38 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=39 + [[ 39 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=40 + [[ 40 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=41 + [[ 41 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=42 + [[ 42 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=43 + [[ 43 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=44 + [[ 44 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=45 + [[ 45 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=46 + [[ 46 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=47 + [[ 47 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=48 + [[ 48 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=49 + [[ 49 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=50 + [[ 50 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=51 + [[ 51 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 1 ]] + echo + wait_pod some-name-pxc-1 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo some-name-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met E0517 02:02:02.392896 24194 reflector.go:227] "Failed to watch" err="Get \"https://34.61.233.25/api/v1/namespaces/sst-retry-limit-3087/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-pxc-1&resourceVersion=1778983318077631020&timeoutSeconds=412&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/some-name-pxc-1 to become Ready.Ok + interrupt_donor_during_sst root_password some-name-pxc-3 1 some-name-pxc-1 some-name-pxc-2 + local root_pass=root_password + local joiner_pod=some-name-pxc-3 + local current_restart_count=1 + local target_restart_count=2 + local donor_pod + donor_candidates=("${@:4}") + local donor_candidates + local retry=0 + local observed_restart_count + echo -n 'waiting for pod/some-name-pxc-3 restartCount=1 SST attempt' waiting for pod/some-name-pxc-3 restartCount=1 SST attempt+ : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=1 + [[ 1 == \1 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + echo + break ++ detect_active_donor root_password 120 some-name-pxc-1 some-name-pxc-2 ++ local root_pass=root_password ++ shift ++ local max_retry=120 ++ shift ++ local retry=0 ++ local donor_pod ++ local donor_rejects_queries ++ local local_state ++ echo -n 'waiting to detect active donor' waiting to detect active donor++ : ++ for donor_pod in '"$@"' +++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace +++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}' ++ donor_rejects_queries=OFF +++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace +++ awk '$1 == "wsrep_local_state_comment" {print $2}' ++ local_state=Donor/Desynced ++ [[ OFF == \O\N ]] ++ [[ Donor/Desynced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]] ++ echo some-name-pxc-1 ++ return 0 + donor_pod=some-name-pxc-1 + echo 'interrupting donor pod/some-name-pxc-1 during SST attempt 2' interrupting donor pod/some-name-pxc-1 during SST attempt 2 + kubectl_bin delete pod some-name-pxc-1 --wait=false --grace-period=0 --force ++ mktemp + local LAST_OUT=/tmp/tmp.YOoCBE4EmO ++ mktemp + local LAST_ERR=/tmp/tmp.way6xAv9Kb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod some-name-pxc-1 --wait=false --grace-period=0 --force + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YOoCBE4EmO pod "some-name-pxc-1" force deleted from sst-retry-limit-3087 namespace + cat /tmp/tmp.way6xAv9Kb Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.YOoCBE4EmO /tmp/tmp.way6xAv9Kb + return 0 + wait_for_pxc_restart_count some-name-pxc-3 2 600 + local pod=some-name-pxc-3 + local expected=2 + local max_retry=600 + local retry=0 + echo -n 'waiting for pod/some-name-pxc-3 restartCount to reach 2' waiting for pod/some-name-pxc-3 restartCount to reach 2++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=7 + [[ 7 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=8 + [[ 8 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=9 + [[ 9 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=10 + [[ 10 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=11 + [[ 11 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=12 + [[ 12 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=13 + [[ 13 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=14 + [[ 14 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=15 + [[ 15 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=16 + [[ 16 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=17 + [[ 17 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=18 + [[ 18 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=19 + [[ 19 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=20 + [[ 20 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=21 + [[ 21 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=22 + [[ 22 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=23 + [[ 23 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=24 + [[ 24 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=25 + [[ 25 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=26 + [[ 26 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=27 + [[ 27 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=28 + [[ 28 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=29 + [[ 29 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=30 + [[ 30 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=31 + [[ 31 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=32 + [[ 32 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=33 + [[ 33 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=34 + [[ 34 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=35 + [[ 35 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=36 + [[ 36 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=37 + [[ 37 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=38 + [[ 38 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=39 + [[ 39 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=40 + [[ 40 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=41 + [[ 41 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=42 + [[ 42 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=43 + [[ 43 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=44 + [[ 44 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=45 + [[ 45 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=46 + [[ 46 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 2 -ge 2 ]] + echo + wait_pod some-name-pxc-1 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo some-name-pxc-1 + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met E0517 02:04:43.172624 14583 reflector.go:227] "Failed to watch" err="Get \"https://34.61.233.25/api/v1/namespaces/sst-retry-limit-3087/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-pxc-1&resourceVersion=1778983478409791020&timeoutSeconds=304&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/some-name-pxc-1 to become Ready.Ok + desc 'verify the joiner stops restarting after the retry limit is reached' + set +o xtrace ----------------------------------------------------------------------------------- verify the joiner stops restarting after the retry limit is reached ----------------------------------------------------------------------------------- + retry=0 + echo -n 'waiting for pod/some-name-pxc-3 logs to contain '\''SST retry limit reached.'\''' waiting for pod/some-name-pxc-3 logs to contain 'SST retry limit reached.'+ grep -q 'SST retry limit reached.' + kubectl_bin logs some-name-pxc-3 -c pxc + echo + desc 'verify retries stop after the limit' + set +o xtrace ----------------------------------------------------------------------------------- verify retries stop after the limit ----------------------------------------------------------------------------------- + local joiner_phase ++ kubectl_bin get pod some-name-pxc-3 -o 'jsonpath={.status.phase}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Fp2evA3vib +++ mktemp ++ local LAST_ERR=/tmp/tmp.cWLTguXKsQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o 'jsonpath={.status.phase}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Fp2evA3vib ++ cat /tmp/tmp.cWLTguXKsQ ++ rm /tmp/tmp.Fp2evA3vib /tmp/tmp.cWLTguXKsQ ++ return 0 + joiner_phase=Running + local joiner_ready ++ jq -r '.status.conditions[] | select(.type == "Ready") | .status' ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.1VUeY6RBCe +++ mktemp ++ local LAST_ERR=/tmp/tmp.0TBZ415Tsf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1VUeY6RBCe ++ cat /tmp/tmp.0TBZ415Tsf ++ rm /tmp/tmp.1VUeY6RBCe /tmp/tmp.0TBZ415Tsf ++ return 0 + joiner_ready=False + [[ Running != \R\u\n\n\i\n\g ]] + [[ False != \F\a\l\s\e ]] + local restart_count_before ++ get_pxc_restart_count some-name-pxc-3 ++ local pod=some-name-pxc-3 ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[] | select(.name == "pxc") | .restartCount' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wWaZcCccMA +++ mktemp ++ local LAST_ERR=/tmp/tmp.hbRNqYbp74 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wWaZcCccMA ++ cat /tmp/tmp.hbRNqYbp74 ++ rm /tmp/tmp.wWaZcCccMA /tmp/tmp.hbRNqYbp74 ++ return 0 + restart_count_before=2 + sleep_with_log 120 + local d=120 + log 'sleeping for 120 seconds' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-05-17T02:05:00+0000]' sleeping for 120 seconds [2026-05-17T02:05:00+0000] sleeping for 120 seconds + sleep 120 + local restart_count_after ++ get_pxc_restart_count some-name-pxc-3 ++ local pod=some-name-pxc-3 ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.PjoUuZalyq +++ mktemp ++ local LAST_ERR=/tmp/tmp.LRuC5dRr2l ++ local exit_status=0 ++ jq -r '.status.containerStatuses[] | select(.name == "pxc") | .restartCount' +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PjoUuZalyq ++ cat /tmp/tmp.LRuC5dRr2l ++ rm /tmp/tmp.PjoUuZalyq /tmp/tmp.LRuC5dRr2l ++ return 0 + restart_count_after=2 + [[ 2 != \2 ]] + desc 'verify donor candidates remain healthy' + set +o xtrace ----------------------------------------------------------------------------------- verify donor candidates remain healthy ----------------------------------------------------------------------------------- + local donor_result + local donor_pod + for donor_pod in '"${donor_candidates[@]}"' ++ run_mysql_local 'SELECT COUNT(*) FROM myApp.myApp;' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 ++ local 'command=SELECT COUNT(*) FROM myApp.myApp;' ++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' ++ local pod=some-name-pxc-1 ++ local container_name= ++ set +o xtrace + donor_result=1 + [[ 1 != \1 ]] + for donor_pod in '"${donor_candidates[@]}"' ++ run_mysql_local 'SELECT COUNT(*) FROM myApp.myApp;' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-2 ++ local 'command=SELECT COUNT(*) FROM myApp.myApp;' ++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' ++ local pod=some-name-pxc-2 ++ local container_name= ++ set +o xtrace + donor_result=1 + [[ 1 != \1 ]] + destroy sst-retry-limit-3087 + local namespace=sst-retry-limit-3087 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'get backup status: Job.batch' + grep -v level=info + sort -u + tee /tmp/tmp.osnukoNZUe/operator.log ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator + grep -v 'the object has been modified' ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ local LAST_OUT=/tmp/tmp.6oadObwdKH +++ mktemp ++ local LAST_ERR=/tmp/tmp.TYPAHGFs6f ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6oadObwdKH ++ cat /tmp/tmp.TYPAHGFs6f ++ rm /tmp/tmp.6oadObwdKH /tmp/tmp.TYPAHGFs6f ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-55d95dc9d8-gmfr6 ++ mktemp + local LAST_OUT=/tmp/tmp.5RTAJBDaDt ++ mktemp + local LAST_ERR=/tmp/tmp.oOK9sjEFB1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-55d95dc9d8-gmfr6 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5RTAJBDaDt + cat /tmp/tmp.oOK9sjEFB1 + rm /tmp/tmp.5RTAJBDaDt /tmp/tmp.oOK9sjEFB1 + return 0 } }, }, { }, }, { }, }, { }, - }, - { - }, + }, ... // 16 identical fields ... // 16 identical fields 2026-05-17T01:52:25.380Z INFO setup Manager starting up {"gitCommit": "6d392bea35083cf1f9dca50d602c31a2f57502ed", "gitBranch": "PR-2473-6d392bea", "buildTime": "2026-05-16T22:17:40Z", "goVersion": "go1.26.3", "os": "linux", "arch": "amd64"} 2026-05-17T01:52:25.380Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.33.11-gke.1137000"} 2026-05-17T01:52:25.381Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-05-17T01:52:25.384Z INFO setup Registering Components. 2026-05-17T01:52:26.398Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-05-17T01:52:26.398Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-05-17T01:52:26.398Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-05-17T01:52:26.398Z INFO controller-runtime.metrics Starting metrics server 2026-05-17T01:52:26.398Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-05-17T01:52:26.398Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-05-17T01:52:26.398Z INFO controller-runtime.webhook Starting webhook server 2026-05-17T01:52:26.398Z INFO setup Starting the Cmd. 2026-05-17T01:52:26.398Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-05-17T01:52:26.499Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-05-17T01:52:26.527Z DEBUG events percona-xtradb-cluster-operator-55d95dc9d8-gmfr6_b49b3cc4-1b4c-478f-89b6-ee08b9e1e54a became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"3ed63feb-17d0-4bb8-af4b-6740386c4ae9","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1778982746521823009"}, "reason": "LeaderElection"} 2026-05-17T01:52:26.527Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-05-17T01:52:26.527Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-05-17T01:52:26.528Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-05-17T01:52:26.528Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-05-17T01:52:26.529Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-05-17T01:52:26.729Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-05-17T01:52:26.729Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-05-17T01:52:26.729Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-05-17T01:52:26.729Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-05-17T01:52:26.730Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-05-17T01:52:26.730Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-05-17T01:53:28.744Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "374f948d-6997-4bcc-8c6f-8ecb15d67765", "version": "1.20.0"} 2026-05-17T01:53:29.189Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "374f948d-6997-4bcc-8c6f-8ecb15d67765", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-05-17T01:53:29.232Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "374f948d-6997-4bcc-8c6f-8ecb15d67765", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-05-17T01:53:29.342Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "374f948d-6997-4bcc-8c6f-8ecb15d67765", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-05-17T01:53:29.376Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "374f948d-6997-4bcc-8c6f-8ecb15d67765", "object": "some-name-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-05-17T01:53:30.093Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "5d831c22-c578-428c-9d26-b65ff02ab94d", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-05-17T01:54:45.853Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "13a02e11-48ef-4f65-82d5-6a27a6d495f9", "user": "operator"} 2026-05-17T01:54:45.885Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "13a02e11-48ef-4f65-82d5-6a27a6d495f9", "user": "monitor"} 2026-05-17T01:54:45.932Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "13a02e11-48ef-4f65-82d5-6a27a6d495f9"} 2026-05-17T01:54:45.966Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "13a02e11-48ef-4f65-82d5-6a27a6d495f9"} 2026-05-17T01:54:45.997Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "13a02e11-48ef-4f65-82d5-6a27a6d495f9", "user": "xtrabackup"} 2026-05-17T01:54:46.037Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "13a02e11-48ef-4f65-82d5-6a27a6d495f9"} 2026-05-17T01:54:46.070Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "13a02e11-48ef-4f65-82d5-6a27a6d495f9", "user": "replication"} 2026-05-17T01:54:46.072Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "13a02e11-48ef-4f65-82d5-6a27a6d495f9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:54:51.161Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "98db3dc6-38ba-4a37-909e-5f0c67a51d9b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:54:56.252Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "f3c8cf82-8149-48b7-b717-966519bd4653", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:55:01.337Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "c0b5770f-b0b9-48f4-91a1-80d8bea4f272", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:55:06.428Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "d3df05d6-e274-4b2b-b2ee-11d98dd811b2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:55:11.516Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "ccdcf71c-4b11-4ca2-981e-221cbdcc606f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:55:16.607Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "0f79e3d4-05de-4298-b0b4-fa279cca926a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:55:21.699Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "ae45bc2b-0957-436e-85c7-c49d9b5dfd10", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:55:26.797Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "51e0f493-a688-4d61-a369-b31a84ec9af5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:55:31.886Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "1a6da0b1-5201-4578-a148-64d673396403", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:55:36.973Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "87f3178b-5966-4238-9c53-46d0490d16d8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:55:42.055Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "f6286b21-eaf6-40e3-982c-4f0d6a82cb67", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:55:47.143Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "f179c19b-3d6d-441d-a0bf-4171cc179d10", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:55:52.234Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "e4463177-f26a-4525-a1b8-9d5fec4a40d1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:55:57.325Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "9d905b2e-6b84-45b6-8560-c5072c59aea1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:56:02.409Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "c261b10a-8fd8-4ee1-8cb5-e1fca3fa94bf", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:56:07.515Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "9962e4ef-4c54-48a0-8e53-de9d4af75e17", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:56:07.665Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "c4c645ab-20b0-45fa-8e5c-052ced1e94ee", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:56:12.781Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "d9155985-ed47-4aad-9844-3cea01e2023f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:56:18.205Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "a5cef96e-bb6b-4ceb-80d9-b5af04d19b3c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:56:23.435Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "b4844d5b-2766-478d-bfdc-56f7e4b2bad7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:56:28.596Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "8e2fc3cd-7124-4486-8523-0736ff2e2951", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:56:33.731Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "f3bff642-cfdf-445d-b4ff-2a1d70c0c9ff", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:56:38.848Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "2ea90b16-e26b-4278-acd7-6c67d775f85c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:56:43.969Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "ae4fa240-0ef9-4bd8-a88d-abe9b1d1732a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:56:49.102Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "e36d82af-568c-47ee-b026-3c2abce74d22", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:56:54.230Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "80813d42-76eb-4a01-9fb0-6164b3a5fc3a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:56:59.360Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "58d5bbe1-f139-4f14-825a-246899200e3e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:57:04.479Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "3b588fbf-fdad-44f2-9418-ff0d3a04b91d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:57:09.605Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "e751187d-cfe9-40d1-a345-351dc9b905d5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:57:14.723Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "2c2a763a-a4cf-4b02-b5a2-03248c503d37", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:57:19.843Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "43c3481c-51d8-4486-9def-e4120a500d71", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:57:24.987Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "84f0b639-98db-4ca6-be7b-e7d671cf6394", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:57:25.701Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "1e91265c-60c3-4eab-ac1f-0492e4105faa", "user": "root"} 2026-05-17T01:57:25.744Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "1e91265c-60c3-4eab-ac1f-0492e4105faa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:57:25.752Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "1e91265c-60c3-4eab-ac1f-0492e4105faa", "new version": "8.0.43-34.1"} 2026-05-17T01:57:26.580Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "1b3731e8-1a90-4b12-81c9-745bee2ac153", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:57:32.251Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "915f7b52-83d9-4ca7-b7d0-d75bacd0e0df", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:57:37.461Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "9236bddf-4cd2-4e5e-9472-85e64803f739", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:57:42.662Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "be748a6a-839a-4f35-9a02-11e03bcc2655", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:57:48.315Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "9d787dec-d06a-4a7d-a8b2-bd1dde1c56a4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:57:53.580Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "44b53f8c-597a-4145-9a5d-cf36cf6cb4cd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:57:58.785Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "425fd906-f3a4-4662-af3f-e27e8e48b290", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:58:03.990Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "8267d401-bf40-404a-b34a-f9eda203b51c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:58:09.183Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "29f2fc3e-fe3e-4ebb-9160-8786a5cbaa40", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:58:14.405Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "8de17b50-6983-4ca4-ba5a-8200ff74f8db", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:58:19.605Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "d36399a5-3fcf-4c96-a9c1-7ab3749718ae", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:58:24.795Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "2152994d-c3a2-44e8-9b72-1f9ee3dd8c14", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:58:29.978Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "4ca9afed-5a93-4562-89da-3310aa55807b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:58:35.165Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "c4b25262-f670-4e00-92cb-f943e2006377", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:58:40.404Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "db1f959f-562e-4665-a542-dfdaddf6447a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:58:45.593Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "4a62479d-6a01-4d25-af4f-e67d08f1a2fa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:58:50.800Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "993081d4-d07c-4394-aedf-d63b29c9c4eb", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:58:56.004Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "3c5c0fb2-90ea-434e-9bbf-f03b397b5af7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:01.225Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "9fb131bd-a6ae-4245-951b-1c1e16f75fcd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:06.440Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "a4935d98-78fb-4852-a7f2-97fe64be59f3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:11.630Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "cbc042e4-93d1-47cb-9560-925abe49d691", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:16.863Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "c59d6e4d-bbf6-430f-902f-fb56463f81d8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:19.142Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "5d654152-ea8b-4ef1-a416-a8e25b67a066", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-05-17T01:59:19.217Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "5d654152-ea8b-4ef1-a416-a8e25b67a066", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-05-17T01:59:19.338Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "5d654152-ea8b-4ef1-a416-a8e25b67a066", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:20.102Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "76ea5dff-b30c-499e-8c9f-0d56b7400849", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:25.671Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "f967f2cb-863e-4dfb-96ca-0bfd04be7c7d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:30.806Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "f800c3d7-ee51-4c10-b328-585c3d655a9e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:35.966Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "d1237c67-6162-455a-b2f1-88b33ca32311", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:41.116Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "90a39d78-c795-4900-b081-8c816ff58c83", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:46.277Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "6482d718-27e7-45e3-a1fc-d9b829572729", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:51.419Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "484e2923-d7b3-495a-8b4f-953d52d0d95f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:56.527Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "9f7448fa-7e69-4f9f-a460-eb97697c9b9c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T01:59:56.653Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "8abce746-6ac8-443b-94dc-6034bff8bfab", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:00:01.780Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "7bb11942-f44d-49fd-9697-a1efb239329c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:00:06.885Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "cc4a2afc-0336-4229-9727-ee2f2b6f48d7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:00:11.998Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "b98f9b2f-daa2-4dd5-9efe-c10002c72c2a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:00:17.098Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "ad4895df-fb30-41a4-9d39-29e93a35a976", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:00:22.189Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "7be9b062-10e2-43b4-8364-608d12ea4ad9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:00:27.300Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "565733d9-b095-4656-a399-47287854f3b4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:00:32.397Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "18038a12-676d-48d9-998f-decd2dca6efa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:00:37.494Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "fe3c2c4b-0d79-4223-8691-7c78c3b6dd75", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:00:42.603Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "3dc7e723-fa05-4fab-b2f8-a94911148c8f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:00:47.981Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "8b092837-3c48-4c93-a63d-8c16319c448c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:00:48.798Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "3c69d8ca-df49-4aa1-9912-2b2f915b230b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:00:54.018Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "5680dc9a-95e0-4f4e-a1c0-5655b0849b8f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:00:59.161Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "11c90914-c4b4-4712-afbb-c7886a5e99fe", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:01:04.311Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "d53aa20b-f867-43c2-ba8f-16a27e24db9e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:01:09.455Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "59e1faa5-ee19-461b-b6ed-5a703c4de0e0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:01:14.606Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "95786dd6-2817-43ac-b79a-40b2890feb25", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:01:19.747Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "c8d2ac12-a2db-400c-8a35-18878e7efd28", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:01:24.884Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "64f41d91-4d44-4d3d-b038-d8ec2154beaf", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:01:30.042Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "73831236-2855-4e85-9a17-306dbc1698a9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:01:35.187Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "2b87180b-be55-456a-902a-2af7ea1e6afe", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:01:40.375Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "fa6b9461-5023-4f27-a669-3a3e8e779423", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:01:45.535Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "27c5bd6c-d365-4aac-bdf3-7f8030ce1edb", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:01:50.684Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "296ddbb4-8033-4a06-900c-eaf45ae1471b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:01:55.841Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "bece7bdd-a601-4e4b-9e25-99a7a5e3fa26", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:02:00.982Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "46c83f6e-3ac4-4a94-8d8b-898162cc9992", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:02:06.125Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "35a9d520-8fe7-4756-86ee-f8533ef6e2aa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:02:11.285Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "5be88ec5-c973-45e0-b50d-de4106f5cd43", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:02:16.508Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "6011f20b-1978-440b-b774-c11ff5c7e1dd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:02:21.630Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "5278953f-116a-4ebd-b08d-39b360ae834e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:02:21.749Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "f6917676-b461-4dcd-ba1e-a17e6aacd095", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:02:26.841Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "745242c0-14d7-408a-807a-5b63a2146486", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:02:31.942Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "f55029dd-c7f9-43c4-8b4b-54c74c513de4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:02:37.049Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "a245433f-b202-4043-b7aa-f8307fbe5f3e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:02:42.142Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "9bf2ae24-8f34-4474-83df-0292cdf7b8ab", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:02:47.251Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "c2309a1c-9584-4c2c-900c-bfe4fc401fa7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:02:52.346Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "8ca78cee-711a-4a8f-b3b4-71e99e1d7389", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:02:57.455Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "f20f37a1-1d8d-4138-886b-585b0fc973b0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:03:02.563Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "e4eb94e7-43f1-4eef-a73b-45d55ae5f061", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:03:07.699Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "9064aecd-b410-4ef4-a0bf-1bf6f3219206", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:03:07.864Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "92ce0157-ccd4-4be6-8b52-68e76f1c8672", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:03:13.544Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "c7725281-d0be-434f-82c5-b41c452fce49", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:03:18.698Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "8715934f-7178-425c-8b7d-be3e24034ad1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:03:23.841Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "c3e4a395-5ace-4fc3-8b7a-9cfc8cb08144", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:03:29.005Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "e6effe02-e9c5-4696-8de7-6aaacc7b348c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:03:34.178Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "c3b392ac-72e5-4f68-a87c-f06e9d0cd1a2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:03:39.325Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "dbff35ab-1c43-4bac-a823-16e57e2b7aa6", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:03:44.475Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "3361f3e2-9fd7-4538-ad9a-8ed67a979d45", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:03:49.619Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "e71de49a-83ea-47e4-baea-b09d2ed31b2b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:03:54.765Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "cd3ab24e-fd00-4150-95ad-98f4d629a4ef", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:03:59.920Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "2c0103b3-e602-4a59-935e-1b48b7c044d2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:04:05.079Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "d64604cc-ed72-467c-b22d-c53841113e43", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:04:10.217Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "2dab306f-8584-4230-ba10-7c49edfed597", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:04:15.361Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "e4642b34-4049-4a35-9258-be58791bcf7c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:04:20.504Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "2293391a-42d0-48b9-a8ca-bd935de7a4e7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:04:25.642Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "8945392c-72cc-4b3d-a17f-d814ed3d8d1a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:04:25.807Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "d9fca206-c024-4423-9883-6fb629afa4ce", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:04:31.496Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "2c1bca15-695a-4fb6-87cb-9961f968b358", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:04:36.663Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "a09f9d3b-d55a-403b-bd10-ef6777eb2694", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:04:41.828Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "bf63c382-fe99-487b-933d-c9db9a5bc045", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:04:42.000Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "9d2cf006-a185-4821-a227-d266eb3742bd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:04:47.745Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "434e1ab9-9d3e-49b3-86e4-f845e5900c1c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:04:53.374Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "69232557-c850-45a1-8ebc-695f8575db53", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:04:58.528Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "ddacebb0-061d-454d-b61b-ffba0c3e20c3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:05:03.703Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "98172587-b7cd-4477-ad71-d1e0896c223a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:05:08.862Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "36c77be9-6681-4091-b2e7-79de5775b989", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:05:14.004Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "5a8c1726-4f04-4091-af8d-a1dadb31b82e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:05:19.177Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "fbb88f6b-c5d5-4a3d-8bed-2f2c1d21bd91", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:05:24.338Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "9a507acc-8120-4c5c-8fd0-18c05c154a24", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:05:29.532Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "d76ca765-430d-4c82-a2eb-60904196bf8c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:05:34.685Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "37f9e135-2130-453b-aacf-ae1bda9ec041", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:05:39.837Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "96b4cf7f-538c-4b69-ac3c-446ab1a40d3b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:05:44.986Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "b6300a2a-6f3f-4712-ba7f-7f6233e5e1c9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:05:50.129Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "979b1b2b-b4ec-45cf-b00e-da9c138fc7e2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:05:55.292Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "427cd56e-06d2-46be-9a52-2411e2045be1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:06:00.445Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "e2018256-a9ef-4261-818d-7f530c5197e0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:06:05.605Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "e2813f61-be5f-4341-9c57-466fe1f03189", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:06:10.779Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "bf9f704e-b1b8-449d-8ceb-1f6445d53221", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:06:15.935Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "3e81c711-2566-478f-b092-f2993a04a9f3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:06:21.089Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "e81575b8-ef08-48d5-b345-ca4a3ec3476a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:06:26.241Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "d7b1ca23-dd1e-4f2d-a600-b72e8d546eaa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:06:31.400Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "756566b2-98b6-4be7-b9b6-28d25a0dae2c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:06:36.562Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "1ade4d96-05d2-47a0-aa43-e41475027c25", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:06:41.741Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "ca49667e-5415-478c-8fbc-435b86ab68dd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:06:46.894Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "d7d067e2-e995-4d4d-badc-64a9174353ce", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:06:52.057Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "cf804f17-8b6f-456f-aa23-272177758f4f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:06:57.212Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "6c4bf477-c09c-4f87-a600-318ee46d5e70", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:07:02.369Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "3076d0af-ba65-4a74-b736-b6aba3ba281b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-05-17T02:07:07.538Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-3087"}, "namespace": "sst-retry-limit-3087", "name": "some-name", "reconcileID": "0722a188-a1ae-42c2-a7e3-a63ed7cf4096", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} ... // 22 identical fields ... // 2 identical fields ... // 3 identical fields ... // 3 identical fields ... // 3 identical fields ... // 4 identical fields ... // 5 identical fields ... // 6 identical fields ... // 7 identical fields ... // 9 identical fields AccessModes: nil, ActiveDeadlineSeconds: nil, Affinity: nil, - Annotations: map[string]string{ + Annotations: map[string]string{ - APIVersion: "apps/v1", - APIVersion: "apps/v1", Args: {"mysqld"}, AutomountServiceAccountToken: nil, + AvailableReplicas: 0, - AvailableReplicas: 3, AWSElasticBlockStore: nil, AzureFile: nil, Capacity: nil, - CollisionCount: &0, + CollisionCount: nil, Conditions: nil, ConfigMap: &v1.ConfigMapVolumeSource{ ContainerPort: 3306, ContainerPort: 33060, ContainerPort: 33062, ContainerPort: 4444, ContainerPort: 4567, ContainerPort: 4568, Containers: []v1.Container{ + CreationTimestamp: v1.Time{}, - CreationTimestamp: v1.Time{Time: s"2026-05-17 01:53:29 +0000 UTC"}, + CurrentReplicas: 0, - CurrentReplicas: 3, + CurrentRevision: "", - CurrentRevision: "some-name-pxc-5cd8885d87", DataSource: nil, DataSourceRef: nil, - DefaultMode: &420, - DefaultMode: &420, + DefaultMode: nil, + DefaultMode: nil, DeletionGracePeriodSeconds: nil, DeletionTimestamp: nil, + DeprecatedServiceAccount: "", - DeprecatedServiceAccount: "default", + DNSPolicy: "", - DNSPolicy: "ClusterFirst", EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "some-name-env-vars-pxc"}, Optional: &true}}}, Env: {{Name: "PXC_SERVICE", Value: "some-name-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name"}, Key: "xtrabackup"}}}, ...}, EphemeralContainers: nil, FailureThreshold: 3, FC: nil, - FieldsType: "FieldsV1", - FieldsType: "FieldsV1", - FieldsV1: s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`..., - FieldsV1: s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`..., Finalizers: nil, + Generation: 0, - Generation: 1, GitRepo: nil, HostAliases: nil, HostIP: "", HostPort: 0, ImagePullPolicy: "IfNotPresent", InitContainers: []v1.Container{ InitialDelaySeconds: 300, ISCSI: nil, Items: nil, Items: nil, Labels: nil, Lifecycle: nil, LivenessProbe: &v1.Probe{ LocalObjectReference: {Name: "auto-some-name-pxc"}, LocalObjectReference: {Name: "some-name-pxc"}, + ManagedFields: nil, - ManagedFields: []v1.ManagedFieldsEntry{ - Manager: "kube-controller-manager", - Manager: "percona-xtradb-cluster-operator", MinReadySeconds: 0, Name: "auto-config", {Name: "bin", VolumeSource: {EmptyDir: &{}}}, Name: "config", Name: "ist", Name: "mysql", Name: "mysql-admin", Name: "mysql-init-file", Name: "mysql-users-secret-file", Name: "mysqlx", Namespace: "sst-retry-limit-3087", Name: "ssl", Name: "ssl-internal", Name: "sst", {Name: "tmp", VolumeSource: {EmptyDir: &{}}}, Name: "vault-keyring-secret", Name: "write-set", NFS: nil, NodeName: "", NodeSelector: nil, ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "cfd6a52398268173b51d7cdb1331c09a", "percona.com/ssl-internal-hash": "751e59a1c1d94ac67f9d6bb6e756acdc"}}, ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, ObjectMeta: v1.ObjectMeta{ + ObservedGeneration: 0, - ObservedGeneration: 1, - Operation: "Update", - Operation: "Update", Optional: &false, Optional: &true, Optional: &true, Ordinals: nil, OS: nil, Overhead: nil, OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "some-name", UID: "73e33f50-4646-4ab6-afef-d95d27ccd9b3", ...}}, - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJl"..., + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6NCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJl"..., + PeriodSeconds: 0, - PeriodSeconds: 10, + PersistentVolumeClaimRetentionPolicy: nil, - PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}", + Phase: "", - Phase: "Pending", + PodManagementPolicy: "", - PodManagementPolicy: "OrderedReady", Ports: []v1.ContainerPort{ PreemptionPolicy: nil, ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}}, + Protocol: "", - Protocol: "TCP", Quobyte: nil, ReadinessProbe: &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...}, + ReadyReplicas: 0, - ReadyReplicas: 3, + Replicas: 0, - Replicas: 3, - Replicas: &3, + Replicas: &4, + ResourceVersion: "", - ResourceVersion: "1778983044452911021", + RestartPolicy: "", - RestartPolicy: "Always", - RevisionHistoryLimit: &10, + RevisionHistoryLimit: nil, + SchedulerName: "", - SchedulerName: "default-scheduler", SecretName: "internal-some-name", SecretName: "some-name-mysql-init", SecretName: "some-name-ssl", SecretName: "some-name-ssl-internal", SecretName: "some-name-vault", Secret: &v1.SecretVolumeSource{ SecurityContext: nil, Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, SelfLink: "", ServiceAccountName: "default", ServiceName: "some-name-pxc", SetHostnameAsFQDN: nil, Spec: v1.PersistentVolumeClaimSpec{ Spec: v1.PodSpec{ Spec: v1.StatefulSetSpec{ StartupProbe: nil, Status: v1.PersistentVolumeClaimStatus{ Status: v1.StatefulSetStatus{ StorageClassName: nil, Subdomain: "", - Subresource: "status", SuccessThreshold: 1, Template: v1.PodTemplateSpec{ TerminationGracePeriodSeconds: &600, TerminationGracePeriodSeconds: nil, + TerminationMessagePath: "", - TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "", - TerminationMessagePolicy: "File", TimeoutSeconds: 5, - Time: s"2026-05-17 01:53:29 +0000 UTC", - Time: s"2026-05-17 01:57:24 +0000 UTC", Tolerations: nil, - TopologySpreadConstraints: nil, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, TypeMeta: {}, TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"}, + UID: "", - UID: "742e91bf-9ea5-4f2f-b370-5afb55753bfb", + UpdatedReplicas: 0, - UpdatedReplicas: 3, + UpdateRevision: "", - UpdateRevision: "some-name-pxc-5cd8885d87", UpdateStrategy: {Type: "RollingUpdate", RollingUpdate: &{Partition: &0}}, &v1.StatefulSet{ VolumeAttributesClassName: nil, VolumeClaimTemplates: []v1.PersistentVolumeClaim{ VolumeDevices: nil, - VolumeMode: &"Filesystem", + VolumeMode: nil, VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...}, VolumeName: "", VolumeSource: v1.VolumeSource{ Volumes: []v1.Volume{ VsphereVolume: nil, WorkingDir: "", + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl get pxc --all-namespaces -o wide + kubectl patch pxc -n sst-retry-limit-3087 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.Iqk0QLxMZ6 ++ mktemp + local LAST_ERR=/tmp/tmp.KR95pGsBYD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Iqk0QLxMZ6 perconaxtradbcluster.pxc.percona.com "some-name" deleted from sst-retry-limit-3087 namespace + cat /tmp/tmp.KR95pGsBYD + rm /tmp/tmp.Iqk0QLxMZ6 /tmp/tmp.KR95pGsBYD + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.WXx6Vy2Qnc ++ mktemp + local LAST_ERR=/tmp/tmp.quvEIFZsH7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WXx6Vy2Qnc No resources found + cat /tmp/tmp.quvEIFZsH7 + rm /tmp/tmp.WXx6Vy2Qnc /tmp/tmp.quvEIFZsH7 + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.lsNbjRreni ++ mktemp + local LAST_ERR=/tmp/tmp.TYA1As0IqD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lsNbjRreni No resources found + cat /tmp/tmp.TYA1As0IqD + rm /tmp/tmp.lsNbjRreni /tmp/tmp.TYA1As0IqD + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.sOHKEBncRN ++ mktemp + local LAST_ERR=/tmp/tmp.nh6PmjeZC7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sOHKEBncRN validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.nh6PmjeZC7 + rm /tmp/tmp.sOHKEBncRN /tmp/tmp.nh6PmjeZC7 + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + rm -rf /tmp/tmp.osnukoNZUe + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator + kubectl_bin delete --grace-period=0 --force=true namespace sst-retry-limit-3087 + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- ++ mktemp + local LAST_OUT=/tmp/tmp.PzvFqEhiMy ++ mktemp + local LAST_OUT=/tmp/tmp.QV45XI8H3t ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.Qej2UtLLAk + local exit_status=0 + local LAST_ERR=/tmp/tmp.idj1HME3je + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace sst-retry-limit-3087 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator