Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/logs/sst-retry-limit-8-4.log
Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1
+ main
+ local cluster=some-name
+ local joiner_pod=some-name-pxc-3
+ donor_candidates=("${cluster}-pxc-1" "${cluster}-pxc-2")
+ local donor_candidates
+ create_infra sst-retry-limit-30476
+ local ns=sst-retry-limit-30476
+ '[' -n pxc-operator ']'
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl get pxc --all-namespaces -o wide
+ kubectl patch pxc -n sst-retry-limit-13801 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched (no change)
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.gQQb0Pf287
++ mktemp
+ local LAST_ERR=/tmp/tmp.LJa4yb9Zjn
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gQQb0Pf287
perconaxtradbcluster.pxc.percona.com "some-name" deleted from sst-retry-limit-13801 namespace
+ cat /tmp/tmp.LJa4yb9Zjn
+ rm /tmp/tmp.gQQb0Pf287 /tmp/tmp.LJa4yb9Zjn
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.WkFTRZFq4D
++ mktemp
+ local LAST_ERR=/tmp/tmp.hnxIiLBf6u
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.WkFTRZFq4D
No resources found
+ cat /tmp/tmp.hnxIiLBf6u
+ rm /tmp/tmp.WkFTRZFq4D /tmp/tmp.hnxIiLBf6u
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.nYAMCplnpG
++ mktemp
+ local LAST_ERR=/tmp/tmp.hD5oGAMZ19
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nYAMCplnpG
No resources found
+ cat /tmp/tmp.hD5oGAMZ19
+ rm /tmp/tmp.nYAMCplnpG /tmp/tmp.hD5oGAMZ19
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ grep chaos-mesh
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ xargs kubectl delete ns
++ mktemp
+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.KVhnwJseoS
++ mktemp
+ local LAST_OUT=/tmp/tmp.M8MwDEDQn5
+ local LAST_ERR=/tmp/tmp.Phh1cPqlcc
+ local exit_status=0
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.zivJ3cNXO1
+ local exit_status=0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.M8MwDEDQn5
+ cat /tmp/tmp.zivJ3cNXO1
+ rm /tmp/tmp.M8MwDEDQn5 /tmp/tmp.zivJ3cNXO1
+ return 0
namespace "sst-retry-limit-13801" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.KVhnwJseoS
namespace "pxc-operator" deleted
+ cat /tmp/tmp.Phh1cPqlcc
+ rm /tmp/tmp.KVhnwJseoS /tmp/tmp.Phh1cPqlcc
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.BIWrDICuIY
++ mktemp
+ local LAST_ERR=/tmp/tmp.npgT7ZK8sB
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.BIWrDICuIY
namespace/pxc-operator created
+ cat /tmp/tmp.npgT7ZK8sB
+ rm /tmp/tmp.BIWrDICuIY /tmp/tmp.npgT7ZK8sB
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TC9D791BLk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.e2Pkv5pLX3
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TC9D791BLk
++ cat /tmp/tmp.e2Pkv5pLX3
++ rm /tmp/tmp.TC9D791BLk /tmp/tmp.e2Pkv5pLX3
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2449-b47912f7-4-cluster8 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZCUriWdVS9
++ mktemp
+ local LAST_ERR=/tmp/tmp.VGzdidF6rD
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2449-b47912f7-4-cluster8 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ZCUriWdVS9
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2449-b47912f7-4-cluster8" modified.
+ cat /tmp/tmp.VGzdidF6rD
+ rm /tmp/tmp.ZCUriWdVS9 /tmp/tmp.VGzdidF6rD
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.uCcDwrt7wQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.tDIeXa4grV
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.uCcDwrt7wQ
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.tDIeXa4grV
+ rm /tmp/tmp.uCcDwrt7wQ /tmp/tmp.tDIeXa4grV
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.6yBy997t1g
++ mktemp
+ local LAST_ERR=/tmp/tmp.6O7Yvxlp17
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6yBy997t1g
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.6O7Yvxlp17
+ rm /tmp/tmp.6yBy997t1g /tmp/tmp.6O7Yvxlp17
+ return 0
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/deploy/cw-operator.yaml
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2449-b47912f7^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.uSd3SC6poq
++ mktemp
+ local LAST_ERR=/tmp/tmp.1m7ebXfd5z
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.uSd3SC6poq
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.1m7ebXfd5z
+ rm /tmp/tmp.uSd3SC6poq /tmp/tmp.1m7ebXfd5z
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.MDOtmZBROa
++ mktemp
+ local LAST_ERR=/tmp/tmp.q0DRXTCWvm
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MDOtmZBROa
pod/percona-xtradb-cluster-operator-7d9c4fb94c-w2hph condition met
+ cat /tmp/tmp.q0DRXTCWvm
+ rm /tmp/tmp.MDOtmZBROa /tmp/tmp.q0DRXTCWvm
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ head -1
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0DxChSytN2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9aNNsF3NFn
++ local exit_status=0
+++ seq 0 2
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0DxChSytN2
++ cat /tmp/tmp.9aNNsF3NFn
++ rm /tmp/tmp.0DxChSytN2 /tmp/tmp.9aNNsF3NFn
++ return 0
+ wait_pod percona-xtradb-cluster-operator-7d9c4fb94c-w2hph 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-7d9c4fb94c-w2hph
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-7d9c4fb94c-w2hph
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-7d9c4fb94c-w2hph condition met
waiting for pod/percona-xtradb-cluster-operator-7d9c4fb94c-w2hph to become Ready.Ok
+ sleep 3
+ create_namespace sst-retry-limit-30476
+ local namespace=sst-retry-limit-30476
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ grep chaos-mesh.org
++ kubectl get crd
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces sst-retry-limit-30476'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces sst-retry-limit-30476
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace sst-retry-limit-30476
++ mktemp
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.QehdGREHYd
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.kDYqepCtQT
++ mktemp
+ local LAST_ERR=/tmp/tmp.h6PqYFJHVm
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.IcSyE3BVWl
+ local exit_status=0
+ xargs kubectl delete ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace sst-retry-limit-30476
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace sst-retry-limit-30476
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.kDYqepCtQT
+ cat /tmp/tmp.h6PqYFJHVm
+ rm /tmp/tmp.kDYqepCtQT /tmp/tmp.h6PqYFJHVm
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace sst-retry-limit-30476
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.QehdGREHYd
+ cat /tmp/tmp.IcSyE3BVWl
Error from server (NotFound): namespaces "sst-retry-limit-30476" not found
+ rm /tmp/tmp.QehdGREHYd /tmp/tmp.IcSyE3BVWl
+ return 1
+ :
+ wait_for_delete namespace/sst-retry-limit-30476
+ local res=namespace/sst-retry-limit-30476
+ echo -n 'waiting for namespace/sst-retry-limit-30476 to be deleted'
waiting for namespace/sst-retry-limit-30476 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "sst-retry-limit-30476" not found
+ desc 'create namespace sst-retry-limit-30476'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace sst-retry-limit-30476
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace sst-retry-limit-30476
++ mktemp
+ local LAST_OUT=/tmp/tmp.6DoklPPqM0
++ mktemp
+ local LAST_ERR=/tmp/tmp.EcbKoBJaVY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace sst-retry-limit-30476
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6DoklPPqM0
namespace/sst-retry-limit-30476 created
+ cat /tmp/tmp.EcbKoBJaVY
+ rm /tmp/tmp.6DoklPPqM0 /tmp/tmp.EcbKoBJaVY
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IQR6jlkg9f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N8RuS1x9Op
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IQR6jlkg9f
++ cat /tmp/tmp.N8RuS1x9Op
++ rm /tmp/tmp.IQR6jlkg9f /tmp/tmp.N8RuS1x9Op
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2449-b47912f7-4-cluster8 --namespace=sst-retry-limit-30476
++ mktemp
+ local LAST_OUT=/tmp/tmp.vmPuod6Cxb
++ mktemp
+ local LAST_ERR=/tmp/tmp.5n3LG5Yxlv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2449-b47912f7-4-cluster8 --namespace=sst-retry-limit-30476
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vmPuod6Cxb
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2449-b47912f7-4-cluster8" modified.
+ cat /tmp/tmp.5n3LG5Yxlv
+ rm /tmp/tmp.vmPuod6Cxb /tmp/tmp.5n3LG5Yxlv
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.uc6Tywx266
++ mktemp
+ local LAST_ERR=/tmp/tmp.KSQ6QzDCSL
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.uc6Tywx266
secret/minio-secret created
secret/aws-s3-secret created
secret/do-spaces-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.KSQ6QzDCSL
+ rm /tmp/tmp.uc6Tywx266 /tmp/tmp.KSQ6QzDCSL
+ return 0
+ desc 'start 3-node cluster with SST retry limit'
+ set +o xtrace


-----------------------------------------------------------------------------------
start 3-node cluster with SST retry limit
-----------------------------------------------------------------------------------

+ spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml 3 10 /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/secrets.yml /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml
+ local cluster=some-name
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.5s8HnfP4aG
++ mktemp
+ local LAST_ERR=/tmp/tmp.DVv8ar5Znf
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5s8HnfP4aG
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.DVv8ar5Znf
+ rm /tmp/tmp.5s8HnfP4aG /tmp/tmp.DVv8ar5Znf
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml
+ local pvc_name=
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ local LAST_OUT=/tmp/tmp.RaCINlrfwS
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2449-b47912f7#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
++ mktemp
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.sst-retry-limit-30476~
+ local LAST_ERR=/tmp/tmp.DJdtWHygEm
+ local exit_status=0
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.RaCINlrfwS
deployment.apps/pxc-client created
+ cat /tmp/tmp.DJdtWHygEm
+ rm /tmp/tmp.RaCINlrfwS /tmp/tmp.DJdtWHygEm
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml
+ local pvc_name=
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml
+ local pvc_name=
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/conf/some-name.yml
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#'
+ local LAST_OUT=/tmp/tmp.ryhFOJnat8
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2449-b47912f7#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.sst-retry-limit-30476~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ local LAST_ERR=/tmp/tmp.DlIdXZ5D6J
+ local exit_status=0
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
++ seq 0 2
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ryhFOJnat8
perconaxtradbcluster.pxc.percona.com/some-name created
+ cat /tmp/tmp.DlIdXZ5D6J
+ rm /tmp/tmp.ryhFOJnat8 /tmp/tmp.DlIdXZ5D6J
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy some-name
++ local target_cluster=some-name
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.nK5PlfV8vk
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.H6vev1dRcx
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.nK5PlfV8vk
+++ cat /tmp/tmp.H6vev1dRcx
+++ rm /tmp/tmp.nK5PlfV8vk /tmp/tmp.H6vev1dRcx
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.8MczKURYfe
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.p9HHMHjwlI
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.8MczKURYfe
+++ cat /tmp/tmp.p9HHMHjwlI
+++ rm /tmp/tmp.8MczKURYfe /tmp/tmp.p9HHMHjwlI
+++ return 0
++ [[ false == \t\r\u\e ]]
++ echo some-name-pxc
+ local proxy=some-name-pxc
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-30476
++ mktemp
+ local LAST_OUT=/tmp/tmp.8Xh1vXmyTZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.3U8XD9K30S
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-30476
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-30476
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n sst-retry-limit-30476
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.8Xh1vXmyTZ
+ cat /tmp/tmp.3U8XD9K30S
error: no matching resources found
+ rm /tmp/tmp.8Xh1vXmyTZ /tmp/tmp.3U8XD9K30S
+ return 1
+ true
+ wait_for_running some-name-pxc 1
+ local name=some-name-pxc
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-0 480
+ local pod=some-name-pxc-0
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-0 condition met
waiting for pod/some-name-pxc-0 to become Ready.Ok
+ wait_for_running some-name-pxc 3
+ local name=some-name-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-0 480
+ local pod=some-name-pxc-0
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-0 condition met
waiting for pod/some-name-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-1 480
+ local pod=some-name-pxc-1
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-1 condition met
waiting for pod/some-name-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-2 480
+ local pod=some-name-pxc-2
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-2 condition met
waiting for pod/some-name-pxc-2 to become Ready.Ok
+ sleep 10
++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TsdzbiY5L7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tWvn42BURA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TsdzbiY5L7
++ cat /tmp/tmp.tWvn42BURA
++ rm /tmp/tmp.TsdzbiY5L7 /tmp/tmp.tWvn42BURA
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h some-name-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dEIy6Kkxzu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bAtLY3hewn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.dEIy6Kkxzu
++ cat /tmp/tmp.bAtLY3hewn
++ rm /tmp/tmp.dEIy6Kkxzu /tmp/tmp.bAtLY3hewn
++ return 0
+ client_pod=pxc-client-56fd5498cd-lwb9t
+ wait_pod pxc-client-56fd5498cd-lwb9t
+ local pod=pxc-client-56fd5498cd-lwb9t
+ local max_retry=480
+ local ns=
++ echo pxc-client-56fd5498cd-lwb9t
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-56fd5498cd-lwb9t condition met
waiting for pod/pxc-client-56fd5498cd-lwb9t to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h some-name-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.I1k6fmSm5G
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y5Grvowa83
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.I1k6fmSm5G
++ cat /tmp/tmp.y5Grvowa83
++ rm /tmp/tmp.I1k6fmSm5G /tmp/tmp.y5Grvowa83
++ return 0
+ client_pod=pxc-client-56fd5498cd-lwb9t
+ wait_pod pxc-client-56fd5498cd-lwb9t
+ local pod=pxc-client-56fd5498cd-lwb9t
+ local max_retry=480
+ local ns=
++ echo pxc-client-56fd5498cd-lwb9t
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-56fd5498cd-lwb9t condition met
waiting for pod/pxc-client-56fd5498cd-lwb9t to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1-84.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Pp7M9YbIde
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dEFMe0mTuK
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Pp7M9YbIde
++ cat /tmp/tmp.dEFMe0mTuK
++ rm /tmp/tmp.Pp7M9YbIde /tmp/tmp.dEFMe0mTuK
++ return 0
+ client_pod=pxc-client-56fd5498cd-lwb9t
+ wait_pod pxc-client-56fd5498cd-lwb9t
+ local pod=pxc-client-56fd5498cd-lwb9t
+ local max_retry=480
+ local ns=
++ echo pxc-client-56fd5498cd-lwb9t
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-56fd5498cd-lwb9t condition met
waiting for pod/pxc-client-56fd5498cd-lwb9t to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.E0OCQMLtee/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.E0OCQMLtee/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.E0OCQMLtee/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1-84.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rsDck2gACZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VVOzXU1HLQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rsDck2gACZ
++ cat /tmp/tmp.VVOzXU1HLQ
++ rm /tmp/tmp.rsDck2gACZ /tmp/tmp.VVOzXU1HLQ
++ return 0
+ client_pod=pxc-client-56fd5498cd-lwb9t
+ wait_pod pxc-client-56fd5498cd-lwb9t
+ local pod=pxc-client-56fd5498cd-lwb9t
+ local max_retry=480
+ local ns=
++ echo pxc-client-56fd5498cd-lwb9t
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-56fd5498cd-lwb9t condition met
waiting for pod/pxc-client-56fd5498cd-lwb9t to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.E0OCQMLtee/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.E0OCQMLtee/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.E0OCQMLtee/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1-84.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rOTSK1eR2N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DeSHuSEWC6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rOTSK1eR2N
++ cat /tmp/tmp.DeSHuSEWC6
++ rm /tmp/tmp.rOTSK1eR2N /tmp/tmp.DeSHuSEWC6
++ return 0
+ client_pod=pxc-client-56fd5498cd-lwb9t
+ wait_pod pxc-client-56fd5498cd-lwb9t
+ local pod=pxc-client-56fd5498cd-lwb9t
+ local max_retry=480
+ local ns=
++ echo pxc-client-56fd5498cd-lwb9t
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-56fd5498cd-lwb9t condition met
waiting for pod/pxc-client-56fd5498cd-lwb9t to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ [[ ! -s /tmp/tmp.E0OCQMLtee/select-1.sql ]]
++ grep 'Unknown MySQL server host' /tmp/tmp.E0OCQMLtee/select-1.sql
+ [[ -n '' ]]
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2449/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.E0OCQMLtee/select-1.sql
+ is_keyring_plugin_in_use some-name
+ local cluster=some-name
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]]
+ kubectl exec some-name-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my
+ return 1
++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.secretsName}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XnqHLi1pwr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AEyNWFXybF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XnqHLi1pwr
++ cat /tmp/tmp.AEyNWFXybF
++ rm /tmp/tmp.XnqHLi1pwr /tmp/tmp.AEyNWFXybF
++ return 0
+ secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BYno6GaOr7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.emIzWijAEc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BYno6GaOr7
++ cat /tmp/tmp.emIzWijAEc
++ rm /tmp/tmp.BYno6GaOr7 /tmp/tmp.emIzWijAEc
++ return 0
+ root_pass=root_password
+ desc 'scale to 4 nodes to trigger SST'
+ set +o xtrace


-----------------------------------------------------------------------------------
scale to 4 nodes to trigger SST
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge -p '{"spec":{"pxc":{"size":4}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.LbpMjTqV6K
++ mktemp
+ local LAST_ERR=/tmp/tmp.YktaEvC4ds
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge -p '{"spec":{"pxc":{"size":4}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.LbpMjTqV6K
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.YktaEvC4ds
+ rm /tmp/tmp.LbpMjTqV6K /tmp/tmp.YktaEvC4ds
+ return 0
+ desc 'verify the joiner pod has SST retry support wired in'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify the joiner pod has SST retry support wired in
-----------------------------------------------------------------------------------

+ local retry
+ retry=0
+ echo -n 'waiting for pod/some-name-pxc-3 container/pxc to start'
waiting for pod/some-name-pxc-3 container/pxc to start+ local joiner_container_started
++ kubectl_bin get pod some-name-pxc-3 -o json
++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OHuVtOAvD1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NHdxg7qwob
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 ']'
++ '[' 1 == 1 ']'
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 ']'
++ '[' 1 == 1 ']'
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.OHuVtOAvD1
++ cat /tmp/tmp.NHdxg7qwob
++ rm /tmp/tmp.OHuVtOAvD1 /tmp/tmp.NHdxg7qwob
++ return 0
+ joiner_container_started=
+ [[ '' == \t\r\u\e ]]
+ sleep 1
+ echo -n .
.+ retry=1
+ [[ 1 -ge 240 ]]
++ kubectl_bin get pod some-name-pxc-3 -o json
++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sqMRsrPPlK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xazbbxgm4r
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sqMRsrPPlK
++ cat /tmp/tmp.xazbbxgm4r
++ rm /tmp/tmp.sqMRsrPPlK /tmp/tmp.xazbbxgm4r
++ return 0
+ joiner_container_started=
+ [[ '' == \t\r\u\e ]]
+ sleep 1
+ echo -n .
.+ retry=2
+ [[ 2 -ge 240 ]]
++ kubectl_bin get pod some-name-pxc-3 -o json
++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8mjOCKOBoz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kdalIYTu6X
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8mjOCKOBoz
++ cat /tmp/tmp.kdalIYTu6X
++ rm /tmp/tmp.8mjOCKOBoz /tmp/tmp.kdalIYTu6X
++ return 0
+ joiner_container_started=false
+ [[ false == \t\r\u\e ]]
+ sleep 1
+ echo -n .
.+ retry=3
+ [[ 3 -ge 240 ]]
++ kubectl_bin get pod some-name-pxc-3 -o json
++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Nbe90xbdrc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nGG2NxmHvY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Nbe90xbdrc
++ cat /tmp/tmp.nGG2NxmHvY
++ rm /tmp/tmp.Nbe90xbdrc /tmp/tmp.nGG2NxmHvY
++ return 0
+ joiner_container_started=false
+ [[ false == \t\r\u\e ]]
+ sleep 1
+ echo -n .
.+ retry=4
+ [[ 4 -ge 240 ]]
++ kubectl_bin get pod some-name-pxc-3 -o json
++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4ZfiJSwmgX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.96UKLxYn0y
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4ZfiJSwmgX
++ cat /tmp/tmp.96UKLxYn0y
++ rm /tmp/tmp.4ZfiJSwmgX /tmp/tmp.96UKLxYn0y
++ return 0
+ joiner_container_started=false
+ [[ false == \t\r\u\e ]]
+ sleep 1
+ echo -n .
.+ retry=5
+ [[ 5 -ge 240 ]]
++ kubectl_bin get pod some-name-pxc-3 -o json
++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.09O7ftuxZ1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WGNJVi4YWb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.09O7ftuxZ1
++ cat /tmp/tmp.WGNJVi4YWb
++ rm /tmp/tmp.09O7ftuxZ1 /tmp/tmp.WGNJVi4YWb
++ return 0
+ joiner_container_started=false
+ [[ false == \t\r\u\e ]]
+ sleep 1
+ echo -n .
.+ retry=6
+ [[ 6 -ge 240 ]]
++ kubectl_bin get pod some-name-pxc-3 -o json
+++ mktemp
++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false'
++ local LAST_OUT=/tmp/tmp.0H3x3PzPVh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wN40Km8O5m
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0H3x3PzPVh
++ cat /tmp/tmp.wN40Km8O5m
++ rm /tmp/tmp.0H3x3PzPVh /tmp/tmp.wN40Km8O5m
++ return 0
+ joiner_container_started=false
+ [[ false == \t\r\u\e ]]
+ sleep 1
+ echo -n .
.+ retry=7
+ [[ 7 -ge 240 ]]
++ kubectl_bin get pod some-name-pxc-3 -o json
++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oHxPPTpyCn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TnfPbt7f0v
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.oHxPPTpyCn
++ cat /tmp/tmp.TnfPbt7f0v
++ rm /tmp/tmp.oHxPPTpyCn /tmp/tmp.TnfPbt7f0v
++ return 0
+ joiner_container_started=false
+ [[ false == \t\r\u\e ]]
+ sleep 1
+ echo -n .
.+ retry=8
+ [[ 8 -ge 240 ]]
++ kubectl_bin get pod some-name-pxc-3 -o json
++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bavpZuD1d7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ht9SHG55sY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bavpZuD1d7
++ cat /tmp/tmp.ht9SHG55sY
++ rm /tmp/tmp.bavpZuD1d7 /tmp/tmp.ht9SHG55sY
++ return 0
+ joiner_container_started=true
+ [[ true == \t\r\u\e ]]
+ echo

+ desc 'interrupt SST twice so the joiner exhausts allowed retries'
+ set +o xtrace


-----------------------------------------------------------------------------------
interrupt SST twice so the joiner exhausts allowed retries
-----------------------------------------------------------------------------------

+ interrupt_donor_during_sst root_password some-name-pxc-3 0 some-name-pxc-1 some-name-pxc-2
+ local root_pass=root_password
+ local joiner_pod=some-name-pxc-3
+ local current_restart_count=0
+ local target_restart_count=1
+ local donor_pod
+ donor_candidates=("${@:4}")
+ local donor_candidates
+ local retry=0
+ local observed_restart_count
+ echo -n 'waiting for pod/some-name-pxc-3 restartCount=0 SST attempt'
waiting for pod/some-name-pxc-3 restartCount=0 SST attempt+ :
++ get_pxc_restart_count some-name-pxc-3
+ observed_restart_count=0
+ [[ 0 == \0 ]]
+ kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress
+ kubectl_bin logs some-name-pxc-3 -c pxc
+ grep -q 'Waiting for SST streaming to complete!'
+ sleep 1
+ echo -n .
.+ retry=1
+ [[ 1 -ge 600 ]]
+ :
++ get_pxc_restart_count some-name-pxc-3
+ observed_restart_count=0
+ [[ 0 == \0 ]]
+ kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress
+ kubectl_bin logs some-name-pxc-3 -c pxc
+ grep -q 'Waiting for SST streaming to complete!'
+ sleep 1
+ echo -n .
.+ retry=2
+ [[ 2 -ge 600 ]]
+ :
++ get_pxc_restart_count some-name-pxc-3
+ observed_restart_count=0
+ [[ 0 == \0 ]]
+ kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress
+ echo

+ break
++ detect_active_donor root_password 120 some-name-pxc-1 some-name-pxc-2
++ local root_pass=root_password
++ shift
++ local max_retry=120
++ shift
++ local retry=0
++ local donor_pod
++ local donor_rejects_queries
++ local local_state
++ echo -n 'waiting to detect active donor'
waiting to detect active donor++ :
++ for donor_pod in '"$@"'
+++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1
+++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}'
+++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';'
+++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
+++ local pod=some-name-pxc-1
+++ local container_name=
+++ set +o xtrace
++ donor_rejects_queries=OFF
+++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1
+++ awk '$1 == "wsrep_local_state_comment" {print $2}'
+++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';'
+++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
+++ local pod=some-name-pxc-1
+++ local container_name=
+++ set +o xtrace
++ local_state=Donor/Desynced
++ [[ OFF == \O\N ]]
++ [[ Donor/Desynced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]]
++ echo some-name-pxc-1
++ return 0
+ donor_pod=some-name-pxc-1
+ echo 'interrupting donor pod/some-name-pxc-1 during SST attempt 1'
interrupting donor pod/some-name-pxc-1 during SST attempt 1
+ kubectl_bin delete pod some-name-pxc-1 --wait=false --grace-period=0 --force
++ mktemp
+ local LAST_OUT=/tmp/tmp.BtvCzD0DOY
++ mktemp
+ local LAST_ERR=/tmp/tmp.qAFymemT4H
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod some-name-pxc-1 --wait=false --grace-period=0 --force
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.BtvCzD0DOY
pod "some-name-pxc-1" force deleted from sst-retry-limit-30476 namespace
+ cat /tmp/tmp.qAFymemT4H
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
+ rm /tmp/tmp.BtvCzD0DOY /tmp/tmp.qAFymemT4H
+ return 0
+ wait_for_pxc_restart_count some-name-pxc-3 1 600
+ local pod=some-name-pxc-3
+ local expected=1
+ local max_retry=600
+ local retry=0
+ echo -n 'waiting for pod/some-name-pxc-3 restartCount to reach 1'
waiting for pod/some-name-pxc-3 restartCount to reach 1++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=1
+ [[ 1 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=2
+ [[ 2 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=3
+ [[ 3 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=4
+ [[ 4 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=5
+ [[ 5 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=6
+ [[ 6 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=7
+ [[ 7 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=8
+ [[ 8 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=9
+ [[ 9 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=10
+ [[ 10 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=11
+ [[ 11 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=12
+ [[ 12 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=13
+ [[ 13 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=14
+ [[ 14 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=15
+ [[ 15 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=16
+ [[ 16 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=17
+ [[ 17 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=18
+ [[ 18 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=19
+ [[ 19 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=20
+ [[ 20 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=21
+ [[ 21 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=22
+ [[ 22 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=23
+ [[ 23 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=24
+ [[ 24 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=25
+ [[ 25 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=26
+ [[ 26 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=27
+ [[ 27 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=28
+ [[ 28 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=29
+ [[ 29 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=30
+ [[ 30 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=31
+ [[ 31 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=32
+ [[ 32 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=33
+ [[ 33 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=34
+ [[ 34 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=35
+ [[ 35 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=36
+ [[ 36 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=37
+ [[ 37 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=38
+ [[ 38 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=39
+ [[ 39 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=40
+ [[ 40 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=41
+ [[ 41 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=42
+ [[ 42 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=43
+ [[ 43 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=44
+ [[ 44 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=45
+ [[ 45 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=46
+ [[ 46 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=47
+ [[ 47 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=48
+ [[ 48 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=49
+ [[ 49 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=50
+ [[ 50 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=51
+ [[ 51 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=52
+ [[ 52 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=53
+ [[ 53 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=54
+ [[ 54 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=55
+ [[ 55 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=56
+ [[ 56 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=57
+ [[ 57 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=58
+ [[ 58 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=59
+ [[ 59 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=60
+ [[ 60 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=61
+ [[ 61 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=62
+ [[ 62 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=63
+ [[ 63 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=64
+ [[ 64 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=65
+ [[ 65 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=66
+ [[ 66 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=67
+ [[ 67 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 0 -ge 1 ]]
+ sleep 1
+ echo -n .
.+ retry=68
+ [[ 68 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 1 ]]
+ echo

+ wait_pod some-name-pxc-1
+ local pod=some-name-pxc-1
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-1 condition met
waiting for pod/some-name-pxc-1 to become Ready.Ok
+ interrupt_donor_during_sst root_password some-name-pxc-3 1 some-name-pxc-1 some-name-pxc-2
+ local root_pass=root_password
+ local joiner_pod=some-name-pxc-3
+ local current_restart_count=1
+ local target_restart_count=2
+ local donor_pod
+ donor_candidates=("${@:4}")
+ local donor_candidates
+ local retry=0
+ local observed_restart_count
+ echo -n 'waiting for pod/some-name-pxc-3 restartCount=1 SST attempt'
waiting for pod/some-name-pxc-3 restartCount=1 SST attempt+ :
++ get_pxc_restart_count some-name-pxc-3
+ observed_restart_count=1
+ [[ 1 == \1 ]]
+ kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress
+ echo

+ break
++ detect_active_donor root_password 120 some-name-pxc-1 some-name-pxc-2
++ local root_pass=root_password
++ shift
++ local max_retry=120
++ shift
++ local retry=0
++ local donor_pod
++ local donor_rejects_queries
++ local local_state
++ echo -n 'waiting to detect active donor'
waiting to detect active donor++ :
++ for donor_pod in '"$@"'
+++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1
+++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}'
+++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';'
+++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
+++ local pod=some-name-pxc-1
+++ local container_name=
+++ set +o xtrace
++ donor_rejects_queries=OFF
+++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1
+++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';'
+++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
+++ local pod=some-name-pxc-1
+++ local container_name=
+++ set +o xtrace
+++ awk '$1 == "wsrep_local_state_comment" {print $2}'
++ local_state=Synced
++ [[ OFF == \O\N ]]
++ [[ Synced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]]
++ for donor_pod in '"$@"'
+++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-2
+++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';'
+++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}'
+++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
+++ local pod=some-name-pxc-2
+++ local container_name=
+++ set +o xtrace
++ donor_rejects_queries=OFF
+++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-2
+++ awk '$1 == "wsrep_local_state_comment" {print $2}'
+++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';'
+++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
+++ local pod=some-name-pxc-2
+++ local container_name=
+++ set +o xtrace
++ local_state=Synced
++ [[ OFF == \O\N ]]
++ [[ Synced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]]
++ sleep 1
++ echo -n .
.++ retry=1
++ [[ 1 -ge 120 ]]
++ :
++ for donor_pod in '"$@"'
+++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1
+++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}'
+++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';'
+++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
+++ local pod=some-name-pxc-1
+++ local container_name=
+++ set +o xtrace
++ donor_rejects_queries=OFF
+++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1
+++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';'
+++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
+++ local pod=some-name-pxc-1
+++ local container_name=
+++ set +o xtrace
+++ awk '$1 == "wsrep_local_state_comment" {print $2}'
++ local_state=Synced
++ [[ OFF == \O\N ]]
++ [[ Synced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]]
++ for donor_pod in '"$@"'
+++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-2
+++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';'
+++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
+++ local pod=some-name-pxc-2
+++ local container_name=
+++ set +o xtrace
+++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}'
++ donor_rejects_queries=OFF
+++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-2
+++ awk '$1 == "wsrep_local_state_comment" {print $2}'
+++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';'
+++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
+++ local pod=some-name-pxc-2
+++ local container_name=
+++ set +o xtrace
++ local_state=Synced
++ [[ OFF == \O\N ]]
++ [[ Synced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]]
++ sleep 1
++ echo -n .
.++ retry=2
++ [[ 2 -ge 120 ]]
++ :
++ for donor_pod in '"$@"'
+++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1
+++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}'
+++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';'
+++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
+++ local pod=some-name-pxc-1
+++ local container_name=
+++ set +o xtrace
++ donor_rejects_queries=OFF
+++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1
+++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';'
+++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
+++ local pod=some-name-pxc-1
+++ local container_name=
+++ set +o xtrace
+++ awk '$1 == "wsrep_local_state_comment" {print $2}'
++ local_state=Donor/Desynced
++ [[ OFF == \O\N ]]
++ [[ Donor/Desynced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]]
++ echo some-name-pxc-1
++ return 0
+ donor_pod=some-name-pxc-1
+ echo 'interrupting donor pod/some-name-pxc-1 during SST attempt 2'
interrupting donor pod/some-name-pxc-1 during SST attempt 2
+ kubectl_bin delete pod some-name-pxc-1 --wait=false --grace-period=0 --force
++ mktemp
+ local LAST_OUT=/tmp/tmp.TRKUFvcqqT
++ mktemp
+ local LAST_ERR=/tmp/tmp.Y0gUoJkHXX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod some-name-pxc-1 --wait=false --grace-period=0 --force
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.TRKUFvcqqT
pod "some-name-pxc-1" force deleted from sst-retry-limit-30476 namespace
+ cat /tmp/tmp.Y0gUoJkHXX
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
+ rm /tmp/tmp.TRKUFvcqqT /tmp/tmp.Y0gUoJkHXX
+ return 0
+ wait_for_pxc_restart_count some-name-pxc-3 2 600
+ local pod=some-name-pxc-3
+ local expected=2
+ local max_retry=600
+ local retry=0
+ echo -n 'waiting for pod/some-name-pxc-3 restartCount to reach 2'
waiting for pod/some-name-pxc-3 restartCount to reach 2++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=1
+ [[ 1 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=2
+ [[ 2 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=3
+ [[ 3 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=4
+ [[ 4 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=5
+ [[ 5 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=6
+ [[ 6 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=7
+ [[ 7 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=8
+ [[ 8 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=9
+ [[ 9 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=10
+ [[ 10 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=11
+ [[ 11 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=12
+ [[ 12 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=13
+ [[ 13 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=14
+ [[ 14 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=15
+ [[ 15 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=16
+ [[ 16 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=17
+ [[ 17 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=18
+ [[ 18 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=19
+ [[ 19 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=20
+ [[ 20 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=21
+ [[ 21 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=22
+ [[ 22 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=23
+ [[ 23 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=24
+ [[ 24 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=25
+ [[ 25 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=26
+ [[ 26 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=27
+ [[ 27 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=28
+ [[ 28 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=29
+ [[ 29 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=30
+ [[ 30 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=31
+ [[ 31 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=32
+ [[ 32 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=33
+ [[ 33 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=34
+ [[ 34 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=35
+ [[ 35 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=36
+ [[ 36 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=37
+ [[ 37 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=38
+ [[ 38 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=39
+ [[ 39 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=40
+ [[ 40 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=41
+ [[ 41 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=42
+ [[ 42 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=43
+ [[ 43 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=44
+ [[ 44 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=45
+ [[ 45 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=46
+ [[ 46 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=47
+ [[ 47 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=48
+ [[ 48 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=49
+ [[ 49 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=50
+ [[ 50 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=51
+ [[ 51 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=52
+ [[ 52 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=53
+ [[ 53 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=54
+ [[ 54 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=55
+ [[ 55 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=56
+ [[ 56 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=57
+ [[ 57 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=58
+ [[ 58 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=59
+ [[ 59 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=60
+ [[ 60 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=61
+ [[ 61 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=62
+ [[ 62 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=63
+ [[ 63 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=64
+ [[ 64 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=65
+ [[ 65 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=66
+ [[ 66 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=67
+ [[ 67 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=68
+ [[ 68 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=69
+ [[ 69 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=70
+ [[ 70 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=71
+ [[ 71 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=72
+ [[ 72 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=73
+ [[ 73 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=74
+ [[ 74 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 1 -ge 2 ]]
+ sleep 1
+ echo -n .
.+ retry=75
+ [[ 75 -ge 600 ]]
++ get_pxc_restart_count some-name-pxc-3
+ [[ 2 -ge 2 ]]
+ echo

+ wait_pod some-name-pxc-1
+ local pod=some-name-pxc-1
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-1 condition met
waiting for pod/some-name-pxc-1 to become Ready.Ok
+ desc 'verify the joiner stops restarting after the retry limit is reached'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify the joiner stops restarting after the retry limit is reached
-----------------------------------------------------------------------------------

+ retry=0
+ echo -n 'waiting for pod/some-name-pxc-3 logs to contain '\''SST retry limit reached.'\'''
waiting for pod/some-name-pxc-3 logs to contain 'SST retry limit reached.'+ kubectl_bin logs some-name-pxc-3 -c pxc
+ grep -q 'SST retry limit reached.'
+ echo

+ desc 'verify retries stop after the limit'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify retries stop after the limit
-----------------------------------------------------------------------------------

+ local joiner_phase
++ kubectl_bin get pod some-name-pxc-3 -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AOLgd1FaKS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zaODSTDklT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AOLgd1FaKS
++ cat /tmp/tmp.zaODSTDklT
++ rm /tmp/tmp.AOLgd1FaKS /tmp/tmp.zaODSTDklT
++ return 0
+ joiner_phase=Running
+ local joiner_ready
++ kubectl_bin get pod some-name-pxc-3 -o json
++ jq -r '.status.conditions[] | select(.type == "Ready") | .status'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9nLGsseiul
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zioXIZ0I7g
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.9nLGsseiul
++ cat /tmp/tmp.zioXIZ0I7g
++ rm /tmp/tmp.9nLGsseiul /tmp/tmp.zioXIZ0I7g
++ return 0
+ joiner_ready=False
+ [[ Running != \R\u\n\n\i\n\g ]]
+ [[ False != \F\a\l\s\e ]]
+ local restart_count_before
++ get_pxc_restart_count some-name-pxc-3
++ local pod=some-name-pxc-3
++ jq -r '.status.containerStatuses[] | select(.name == "pxc") | .restartCount'
++ kubectl_bin get pod some-name-pxc-3 -o json
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HpoyLCvPrQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TlrCGPFrYY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.HpoyLCvPrQ
++ cat /tmp/tmp.TlrCGPFrYY
++ rm /tmp/tmp.HpoyLCvPrQ /tmp/tmp.TlrCGPFrYY
++ return 0
+ restart_count_before=2
+ sleep_with_log 120
+ local d=120
+ log 'sleeping for 120 seconds'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-30T14:38:56+0000]' sleeping for 120 seconds
[2026-04-30T14:38:56+0000] sleeping for 120 seconds
+ sleep 120
+ local restart_count_after
++ get_pxc_restart_count some-name-pxc-3
++ local pod=some-name-pxc-3
++ kubectl_bin get pod some-name-pxc-3 -o json
++ jq -r '.status.containerStatuses[] | select(.name == "pxc") | .restartCount'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U5OsEItC7j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CVsiomqzPJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod some-name-pxc-3 -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.U5OsEItC7j
++ cat /tmp/tmp.CVsiomqzPJ
++ rm /tmp/tmp.U5OsEItC7j /tmp/tmp.CVsiomqzPJ
++ return 0
+ restart_count_after=2
+ [[ 2 != \2 ]]
+ desc 'verify donor candidates remain healthy'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify donor candidates remain healthy
-----------------------------------------------------------------------------------

+ local donor_result
+ local donor_pod
+ for donor_pod in '"${donor_candidates[@]}"'
++ run_mysql_local 'SELECT COUNT(*) FROM myApp.myApp;' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1
++ local 'command=SELECT COUNT(*) FROM myApp.myApp;'
++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
++ local pod=some-name-pxc-1
++ local container_name=
++ set +o xtrace
+ donor_result=1
+ [[ 1 != \1 ]]
+ for donor_pod in '"${donor_candidates[@]}"'
++ run_mysql_local 'SELECT COUNT(*) FROM myApp.myApp;' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-2
++ local 'command=SELECT COUNT(*) FROM myApp.myApp;'
++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\'''
++ local pod=some-name-pxc-2
++ local container_name=
++ set +o xtrace
+ donor_result=1
+ [[ 1 != \1 ]]
+ destroy sst-retry-limit-30476
+ local namespace=sst-retry-limit-30476
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
+ grep -v level=info
+ grep -v 'the object has been modified'
+ grep -v 'get backup status: Job.batch'
+ sort -u
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+ tee /tmp/tmp.E0OCQMLtee/operator.log
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ head -1
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zQns9QUuHd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Sq14MtApQk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zQns9QUuHd
++ cat /tmp/tmp.Sq14MtApQk
++ rm /tmp/tmp.zQns9QUuHd /tmp/tmp.Sq14MtApQk
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-7d9c4fb94c-w2hph
++ mktemp
+ local LAST_OUT=/tmp/tmp.RJ8lTRjHc1
++ mktemp
+ local LAST_ERR=/tmp/tmp.ToIlytvm8y
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-7d9c4fb94c-w2hph
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.RJ8lTRjHc1
+ cat /tmp/tmp.ToIlytvm8y
+ rm /tmp/tmp.RJ8lTRjHc1 /tmp/tmp.ToIlytvm8y
+ return 0

  }
  	},
  		},
  			{
  			},
  				},
  					{
  					},
  						},
  							{
  							},
- 		},
- 			{
- 			},
+ 		},
  						... // 16 identical fields
  							... // 16 identical fields
2026-04-30T14:27:18.631Z	INFO	setup	Feature gates	{"PXCO_FEATURE_GATES": "", "enabled": ""}
2026-04-30T14:27:18.631Z	INFO	setup	Manager starting up	{"gitCommit": "b47912f7729e2d124f0221cb70713ba4110c1fe4", "gitBranch": "PR-2449-b47912f7", "buildTime": "2026-04-30T10:36:22Z", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"}
2026-04-30T14:27:18.631Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.32.13-gke.1362000"}
2026-04-30T14:27:18.634Z	INFO	setup	Registering Components.
2026-04-30T14:27:19.597Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2026-04-30T14:27:19.597Z	INFO	controller-runtime.metrics	Starting metrics server
2026-04-30T14:27:19.597Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2026-04-30T14:27:19.597Z	INFO	setup	Starting the Cmd.
2026-04-30T14:27:19.597Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2026-04-30T14:27:19.598Z	INFO	controller-runtime.certwatcher	Starting certificate poll+watcher	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"}
2026-04-30T14:27:19.598Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"}
2026-04-30T14:27:19.598Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2026-04-30T14:27:19.598Z	INFO	controller-runtime.webhook	Starting webhook server
2026-04-30T14:27:19.699Z	INFO	Attempting to acquire leader lease...	{"lock": "pxc-operator/08db1feb.percona.com"}
2026-04-30T14:27:19.727Z	DEBUG	events	percona-xtradb-cluster-operator-7d9c4fb94c-w2hph_bed8a17e-6b0d-4a3c-b802-131bf8fa9f9b became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"baf684d0-6338-4410-a3cf-fcea0d61d890","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1777559239721103009"}, "reason": "LeaderElection"}
2026-04-30T14:27:19.727Z	INFO	Starting EventSource	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"}
2026-04-30T14:27:19.727Z	INFO	Successfully acquired lease	{"lock": "pxc-operator/08db1feb.percona.com"}
2026-04-30T14:27:19.728Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2026-04-30T14:27:19.728Z	INFO	Starting EventSource	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"}
2026-04-30T14:27:19.728Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2026-04-30T14:27:19.828Z	INFO	Starting Controller	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"}
2026-04-30T14:27:19.828Z	INFO	Starting Controller	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"}
2026-04-30T14:27:19.828Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1}
2026-04-30T14:27:19.828Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1}
2026-04-30T14:27:19.928Z	INFO	Starting Controller	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"}
2026-04-30T14:27:19.928Z	INFO	Starting workers	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1}
2026-04-30T14:28:13.301Z	INFO	Set CR version	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "440f40de-9715-4efe-b58a-e5e4001c95f3", "version": "1.20.0"}
2026-04-30T14:28:13.748Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "440f40de-9715-4efe-b58a-e5e4001c95f3", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-04-30T14:28:13.794Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "440f40de-9715-4efe-b58a-e5e4001c95f3", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-04-30T14:28:13.856Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "440f40de-9715-4efe-b58a-e5e4001c95f3", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-30T14:28:13.893Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "440f40de-9715-4efe-b58a-e5e4001c95f3", "object": "some-name-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-30T14:28:14.650Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "0a204764-7747-44d2-9c5a-062a51b3cb94", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-04-30T14:29:35.918Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "89b27bfe-8fa4-4f90-bb38-117760fcdad5", "user": "operator"}
2026-04-30T14:29:35.955Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "89b27bfe-8fa4-4f90-bb38-117760fcdad5", "user": "monitor"}
2026-04-30T14:29:36.015Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "89b27bfe-8fa4-4f90-bb38-117760fcdad5"}
2026-04-30T14:29:36.062Z	INFO	monitor user privileges granted	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "89b27bfe-8fa4-4f90-bb38-117760fcdad5"}
2026-04-30T14:29:36.104Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "89b27bfe-8fa4-4f90-bb38-117760fcdad5", "user": "xtrabackup"}
2026-04-30T14:29:36.157Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "89b27bfe-8fa4-4f90-bb38-117760fcdad5"}
2026-04-30T14:29:36.197Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "89b27bfe-8fa4-4f90-bb38-117760fcdad5", "user": "replication"}
2026-04-30T14:29:36.200Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "89b27bfe-8fa4-4f90-bb38-117760fcdad5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:29:41.305Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "18e27bc6-5fdb-42f4-a3eb-1a5ea69eede0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:29:46.407Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "61347306-54a5-4df0-bbbc-a2d16b1f7f23", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:29:51.536Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "0531dd21-4565-4d94-8400-1a54b727c4ed", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:29:56.635Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "addba35c-03ae-4c12-8ce6-a8ec5956f5af", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:30:01.745Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "a8489ad8-67b3-4ebe-9dd1-a02a3c8f5a39", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:30:06.855Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "f4e2f610-ffdb-4ee9-afcd-bad0189d2742", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:30:11.954Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "2c4f9889-0c24-4cab-8e55-00aa5afeeb21", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:30:17.068Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "25bcb876-cac4-464b-973f-c25d8eb31e50", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:30:22.170Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "0e4ab976-8af5-4f55-b4ba-eab2a8ba7208", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:30:27.281Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "902a8bf9-d454-4fbf-8acd-06459d27d06c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:30:32.393Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "ea718e59-b859-4275-bb4c-84cf94edc8d5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:30:37.510Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "03452ccf-06b9-4935-9242-26a553ceb2f2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:30:42.624Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "448aba6c-5315-49b3-aa03-6a8e12a95f38", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:30:47.742Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "57728467-7bb3-449e-a31c-d196f7a08b55", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:30:52.851Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "6c7a55fd-b431-4bec-8402-95f44d00cb0e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:30:57.966Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "553adb87-c007-489a-bdf2-a5649777b9d4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:30:58.131Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "f390c835-9f65-40bb-8a06-f89a1f60a641", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:31:03.346Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "c29e507a-a87e-43e7-ba27-14a4fa2e805d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:31:08.520Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "0ba1bf4e-d7b6-402d-adb1-b9b38e3f275f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:31:13.663Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "e151300f-23a4-48b2-b023-51c8ede93d91", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:31:18.827Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "2eefc032-9e6c-460f-acac-cc17dde05811", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:31:24.000Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "3022ce6f-82a0-4362-ada3-334a0e65be6f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:31:29.158Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "4f9f3594-5e2a-4053-afd2-c1da0002f665", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:31:34.318Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "803a188f-00fd-4201-a691-8459787d23d6", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:31:39.448Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "34ae2d46-7c8d-428e-84e8-d27e6a373286", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:31:44.610Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "f100b526-f467-4999-b66e-71790ce36f90", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:31:49.765Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "b631e566-9c03-4359-b88b-da7f75be38c9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:31:54.913Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "5d82fe4a-b7be-4046-9c8d-1df97dc4f51e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:00.096Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "e85a0cbc-95a7-432f-bc52-cd23b71d4890", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:05.281Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "cec2c877-6b59-4e03-a741-dbcd70b1bf82", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:10.446Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "938f5a20-ae0f-4a96-800a-6c4cfe81e956", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:15.633Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "b0a87227-8213-474b-9984-18604e3f0192", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:16.429Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "d00227c1-f5ec-412f-8af3-147bae632fb3", "user": "root"}
2026-04-30T14:32:16.481Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "d00227c1-f5ec-412f-8af3-147bae632fb3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:16.490Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "d00227c1-f5ec-412f-8af3-147bae632fb3", "new version": "8.4.7-7.1"}
2026-04-30T14:32:17.191Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "a4d1d11e-a7eb-4c33-a7cd-46a6e9b72cb2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:22.893Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "0d350a36-d686-469e-aea7-3413803c63da", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:28.111Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "0d2c3104-70f8-47c5-b4dc-3687dac85132", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:33.389Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "3fcbda42-3b29-4110-b842-d43ed6a4732c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:38.610Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "6eb35c82-12e5-46ef-af82-229c254eb332", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:43.822Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "43ffc41e-9bba-4bc7-8dca-ef0b6572f297", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:49.050Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "fc8ded85-39fc-40a9-aaab-da2bfd4ef1a8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:54.297Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "e5f2db71-df4c-4205-94a7-f5bc77da09fd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:32:59.542Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "339e554b-eb74-41c0-a590-325dbf76a72b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:33:04.753Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "d4f9b008-05d7-4dde-bd60-110122a523a8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:33:09.969Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "04c6a6f3-10f9-4caa-9575-9600a965ad8d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:33:15.222Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "8f8d32b3-c6e7-4c77-96d4-989fd855debc", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:33:20.447Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "0904754c-6b4c-4422-9a8b-d4129d276355", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:33:25.694Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "a19484a5-b105-4569-b884-ca5102fc8213", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:33:28.109Z	DEBUG	Updating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "b0eee118-8a81-4db6-ae0b-8a5195b57211", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2026-04-30T14:33:28.599Z	DEBUG	Updating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "b0eee118-8a81-4db6-ae0b-8a5195b57211", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2026-04-30T14:33:28.704Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "b0eee118-8a81-4db6-ae0b-8a5195b57211", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:33:29.530Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "0e7876d3-89cd-4294-b6bd-27f5599f7caa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:33:34.696Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "19aa9d74-7eef-498c-90fc-9add1640650f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:33:39.855Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "a349917b-e53d-43ff-acae-1ce471ee6756", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:33:45.037Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "734630ce-ebe7-4998-9b33-25395827de17", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:33:50.219Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "755aed4d-91d9-4212-a4da-79a3edc7821a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:33:55.409Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "54c5e521-d775-445d-997d-e5ba6821ecde", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:00.572Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "7257185e-f8a6-4bab-8f00-c7a01c6ecf37", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:05.745Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "2987c3fe-fad7-4821-bd6a-5b547c03e379", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:10.895Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "8d905319-cc10-484d-b81c-6e33c15c9df3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:11.036Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "991b03f9-25b9-40c3-bfd6-685aa06a2f99", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:16.156Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "80b9dab2-ce16-428d-bb73-49b5780bc460", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:21.271Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "f785747e-b739-4d84-a455-b78fd6a41616", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:26.387Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "01349a10-34b4-463f-9489-cbbd093e0bb5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:31.503Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "464208dc-60a6-47eb-9b3f-1628859c2e1c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:36.626Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "e128d3db-9d0e-4867-9382-0340056a5a47", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:41.779Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "2ec8508c-cffc-4c11-ad5f-935b33b4471d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:46.882Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "9826a50c-c308-4133-9ac4-01c6eb504825", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:51.988Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "693f3f02-1943-42c9-b21b-641cd945cd16", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:57.161Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "537ed203-bd4a-4bee-a9be-bb2d0dfa8082", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:34:57.416Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "8cbae9a7-2ff4-4d67-a9cf-43edd1f0d57c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:35:03.251Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "e15e9359-9bcb-48dd-9441-c7eecb1a2468", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:35:08.467Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "f8e60ba1-b082-4f20-9a15-85c11d198519", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:35:13.638Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "2641722e-a8d2-4ab1-9297-d03a073294be", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:35:18.883Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "2b65ad29-0a9e-44c0-845b-1ebfbb38c246", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:35:24.147Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "058a081a-d7d2-4207-b222-fb659e5acf39", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:35:29.367Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "e1592815-10f2-45da-9e6c-d513c7536a26", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:35:34.536Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "15dd8bd2-16b4-4045-b78e-83eb5bdbfca7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:35:39.731Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "8f579cf7-11e4-4cb9-a8f4-c53c1880d214", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:35:44.936Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "50eb512f-70dd-484b-9c77-74c000402a7f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:35:50.148Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "2ff63a23-1014-49b6-8dd2-2150984a7af9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:35:55.345Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "1d998d20-df46-43eb-befa-bcd4993dcd10", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:36:00.540Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "7116b31f-cb94-4080-83bb-e80af038439e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:36:05.717Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "6d4b0fe5-7d5b-4ef3-a76d-1e69f3c66df0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:36:10.958Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "09afd1d1-5473-4745-a069-4fdd877edb91", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:36:16.148Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "05298ca2-8cc7-4f8b-a75e-360ed7cecf12", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:36:21.343Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "1eb5e427-cc11-47ca-be70-d4d07e6775a1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:36:26.523Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "f6c64e2d-1154-4460-a6ac-f963e416cbf8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:36:31.878Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "7e90013c-a6f9-4720-96a3-ec748c6e020a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:36:32.199Z	ERROR	Update status	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "7e90013c-a6f9-4720-96a3-ec748c6e020a", "error": "get pxc status: parse some-name-pxc-1 pod logs: get logs from some-name-pxc-1 pod: failed to get pod logs: get pod logs stream: pods \"some-name-pxc-1\" not found", "errorVerbose": "pods \"some-name-pxc-1\" not found\nget pod logs stream\ngithub.com/percona/percona-xtradb-cluster-operator/clientcmd.(*Client).PodLogs.func2\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/clientcmd/clientcmd.go:59\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.35.4/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.4/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.4/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.35.4/util/retry/util.go:50\ngithub.com/percona/percona-xtradb-cluster-operator/clientcmd.(*Client).PodLogs\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/clientcmd/clientcmd.go:54\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isPodWaitingForRecovery\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/full_crash_recovery.go:56\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).appStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:247\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:105\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:257\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:489\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to get pod logs\ngithub.com/percona/percona-xtradb-cluster-operator/clientcmd.(*Client).PodLogs\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/clientcmd/clientcmd.go:73\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isPodWaitingForRecovery\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/full_crash_recovery.go:56\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).appStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:247\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:105\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:257\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:489\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nget logs from some-name-pxc-1 pod\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isPodWaitingForRecovery\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/full_crash_recovery.go:58\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).appStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:247\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:105\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:257\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:489\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nparse some-name-pxc-1 pod logs\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).appStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:249\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:105\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:257\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:489\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nget pxc status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:107\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:257\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:489\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2026-04-30T14:36:37.231Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "e6c81287-4ec8-4c7b-b443-9d8dfae32010", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:36:37.366Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "b44c3fb1-784b-4a83-924c-44783ffd89db", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:36:42.475Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "581f653d-0a1f-4c9f-9656-d5170e1f0146", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:36:47.591Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "9b69eb36-a6f7-4498-af51-a197df920867", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:36:52.718Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "03a17e95-3ac9-45a1-9733-9ecd00f74ace", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:36:57.837Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "34c9a4e1-aa68-46c3-9015-46f1db0566e0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:37:03.065Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "e96fca04-4248-430f-a4b4-9490d39ea771", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:37:08.368Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "ddd395a4-90ca-408a-9b41-f6c7552c5908", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:37:13.484Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "653f507e-eedf-464f-9f78-a2d6621bcd41", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:37:18.618Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "943747e8-54a7-487d-9d25-a72fc10de936", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:37:23.819Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "0dc6296a-93d2-4f28-a2af-83566ad29058", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:37:29.002Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "06f001ce-1160-43c1-820f-ae0be77dc0d7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:37:29.253Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "07c0673a-9666-44a3-9536-356655cb3d1e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:37:34.884Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "cd313acd-4393-4b1a-be24-4faf88bb7e59", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:37:40.355Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "cc3cefa1-7ee5-4f67-bb01-8f866a8e678d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:37:45.702Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "41ec2143-c751-4481-ab2e-d988fe10c2d5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:37:50.896Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "04a09bfc-9da5-4e7e-9409-64d94b4d9dfb", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:37:56.074Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "932d1d9a-451d-402d-90cb-751201d2fa24", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:01.286Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "46456c4f-9d48-48f6-92be-d61ddd6ba716", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:06.478Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "89547438-f7bc-4168-8552-7805a73874d7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:11.648Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "50c9f31b-791f-451b-bdbe-74daf81199fb", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:16.856Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "efd13094-9028-4d5e-abcf-40d048545ed8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:22.069Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "2053a5c9-b155-4214-8cf9-8278e812ca3d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:27.367Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "69512dcf-5f16-4f93-9608-bdbfbc4182d3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:32.549Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "b467c007-35b5-4ece-8be6-d08f6f045761", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:37.713Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "847f16bd-645c-4305-823c-0da75a634c06", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:37.899Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "0747d5e1-7346-4cc0-bbc3-cfe24e5192dc", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:43.565Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "3343987e-18bf-46c3-9833-75fc7c39b256", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:48.742Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "462facc0-5716-4aa6-96c5-ad34b7b7a30f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:48.962Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "161724a3-abf8-4d21-a28f-dbbc9380ec08", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:54.588Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "ff3377ab-ff78-4e41-9a58-d2d6868be678", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:38:59.753Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "9cba868d-387e-4e5f-b69a-ed3122748a99", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:39:04.942Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "6da15fea-22fd-498b-9453-3926276ba070", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:39:10.152Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "a4394c26-eff1-439a-bcd9-60caee2356c3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:39:15.400Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "1cb10ae2-6ed1-40ae-ac32-8f08cd83853f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:39:20.585Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "e0601ebd-0056-4751-b07a-ee530dd13d6d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:39:25.777Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "683af57f-c915-487a-836a-23da2a696761", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:39:30.984Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "8280d7cc-01c6-4965-b305-9b3b59ea7238", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:39:36.168Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "a941da88-8d16-418d-b191-92c57cfed39d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:39:41.377Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "076c503f-79aa-4687-88a2-722df32ea9e9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:39:46.551Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "4db7bc41-74f6-4825-8824-f72ff04b2e53", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:39:51.721Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "d322b755-c8d7-4dde-b844-588dcf824cba", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:39:56.888Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "d6241375-6196-477c-a8a4-1deeb4bd1f31", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:40:02.058Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "c7191ffa-e918-40fc-bded-8f311e701de4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:40:07.233Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "beb1b53e-e948-4073-b788-d59e42619cb9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:40:12.421Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "c22314a8-fbf7-46cc-aa8c-c255b203c31f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:40:17.582Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "ebbf8908-e67b-4872-9670-2c533f0a3637", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:40:22.812Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "ab1ee074-e2f9-40fc-abe9-05e1ebb84943", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:40:28.053Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "bd964612-271b-449c-978f-5ff2bedd969e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:40:33.509Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "9fe08410-40b8-4edd-8aba-f7767ec3eeef", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:40:38.729Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "6d3fd59a-0160-4709-8863-1e6a6d605857", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:40:43.922Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "f26e9cb8-b46e-4085-9152-6380fd317678", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:40:49.089Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "a8f60d79-56b0-4449-b2da-b4eebbcb98c6", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:40:54.313Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "e697797d-9bba-4554-885e-7cd193b55063", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
2026-04-30T14:40:59.476Z	INFO	reconcile replication error	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-30476"}, "namespace": "sst-retry-limit-30476", "name": "some-name", "reconcileID": "cf6e2bcb-eeb2-4e5c-a672-50a0a2efa5a1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"}
  							... // 22 identical fields
  		... // 2 identical fields
  					... // 3 identical fields
  						... // 3 identical fields
  							... // 3 identical fields
  						... // 4 identical fields
  					... // 5 identical fields
  				... // 6 identical fields
  				... // 7 identical fields
  							... // 9 identical fields
  					AccessModes: nil,
  				ActiveDeadlineSeconds:         nil,
  				Affinity:      nil,
- 		Annotations: map[string]string{
+ 		Annotations: map[string]string{
- 				APIVersion:  "apps/v1",
- 				APIVersion: "apps/v1",
  						Args:       {"mysqld"},
  				AutomountServiceAccountToken:  nil,
+ 		AvailableReplicas:  0,
- 		AvailableReplicas:  3,
  							AWSElasticBlockStore: nil,
  							AzureFile: nil,
  					Capacity:    nil,
- 		CollisionCount:     &0,
+ 		CollisionCount:     nil,
  		Conditions:         nil,
  							ConfigMap: &v1.ConfigMapVolumeSource{
  								ContainerPort: 3306,
  								ContainerPort: 33060,
  								ContainerPort: 33062,
  								ContainerPort: 4444,
  								ContainerPort: 4567,
  								ContainerPort: 4568,
  				Containers: []v1.Container{
+ 		CreationTimestamp:          v1.Time{},
- 		CreationTimestamp:          v1.Time{Time: s"2026-04-30 14:28:13 +0000 UTC"},
+ 		CurrentReplicas:    0,
- 		CurrentReplicas:    3,
+ 		CurrentRevision:    "",
- 		CurrentRevision:    "some-name-pxc-89f54f49d",
  					DataSource:                nil,
  					DataSourceRef:             nil,
- 								DefaultMode:          &420,
- 								DefaultMode: &420,
+ 								DefaultMode:          nil,
+ 								DefaultMode: nil,
  		DeletionGracePeriodSeconds: nil,
  		DeletionTimestamp:          nil,
+ 				DeprecatedServiceAccount:      "",
- 				DeprecatedServiceAccount:      "default",
+ 				DNSPolicy:                     "",
- 				DNSPolicy:                     "ClusterFirst",
  						EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "some-name-env-vars-pxc"}, Optional: &true}}},
  						Env:     {{Name: "PXC_SERVICE", Value: "some-name-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name"}, Key: "xtrabackup"}}}, ...},
  				EphemeralContainers:           nil,
  							FailureThreshold:              3,
  							FC:        nil,
- 				FieldsType:  "FieldsV1",
- 				FieldsType: "FieldsV1",
- 				FieldsV1:   s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`...,
- 				FieldsV1:    s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`...,
  		Finalizers:      nil,
+ 		Generation:                 0,
- 		Generation:                 1,
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1
  							GitRepo:              nil,
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:259
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:489
  				HostAliases:   nil,
  								HostIP:        "",
  								HostPort:      0,
  						ImagePullPolicy:          "IfNotPresent",
  				InitContainers: []v1.Container{
  							InitialDelaySeconds:           300,
  							ISCSI: nil,
  								Items:                nil,
  								Items:       nil,
  		Labels:                     nil,
  						Lifecycle:                nil,
  						LivenessProbe: &v1.Probe{
  								LocalObjectReference: {Name: "auto-some-name-pxc"},
  								LocalObjectReference: {Name: "some-name-pxc"},
+ 		ManagedFields: nil,
- 		ManagedFields: []v1.ManagedFieldsEntry{
- 				Manager:     "kube-controller-manager",
- 				Manager:    "percona-xtradb-cluster-operator",
  		MinReadySeconds:                      0,
  						Name: "auto-config",
  					{Name: "bin", VolumeSource: {EmptyDir: &{}}},
  						Name: "config",
  								Name:          "ist",
  								Name:          "mysql",
  								Name:          "mysql-admin",
  						Name: "mysql-init-file",
  						Name: "mysql-users-secret-file",
  								Name:          "mysqlx",
  		Namespace:                  "sst-retry-limit-30476",
  						Name: "ssl",
  						Name: "ssl-internal",
  								Name:          "sst",
  					{Name: "tmp", VolumeSource: {EmptyDir: &{}}},
  						Name: "vault-keyring-secret",
  								Name:          "write-set",
  							NFS:   nil,
  				NodeName:                      "",
  				NodeSelector:                  nil,
  			ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "cfd6a52398268173b51d7cdb1331c09a", "percona.com/ssl-internal-hash": "751e59a1c1d94ac67f9d6bb6e756acdc"}},
  				ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},
  	ObjectMeta: v1.ObjectMeta{
+ 		ObservedGeneration: 0,
- 		ObservedGeneration: 1,
- 				Operation:   "Update",
- 				Operation:  "Update",
  								Optional:    &false,
  								Optional:             &true,
  								Optional:    &true,
  		Ordinals:                             nil,
  				OS:                        nil,
  				Overhead:                  nil,
  		OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "some-name", UID: "7be6d9e3-57e9-4bae-bdd2-372af29f481f", ...}},
- 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJl"...,
+ 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6NCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJl"...,
+ 							PeriodSeconds:                 0,
- 							PeriodSeconds:                 10,
+ 		PersistentVolumeClaimRetentionPolicy: nil,
- 		PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}",
+ 					Phase:       "",
- 					Phase:       "Pending",
+ 		PodManagementPolicy:                  "",
- 		PodManagementPolicy:                  "OrderedReady",
  						Ports: []v1.ContainerPort{
  				PreemptionPolicy:          nil,
  							ProbeHandler:                  {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}},
+ 								Protocol:      "",
- 								Protocol:      "TCP",
  							Quobyte:       nil,
  						ReadinessProbe:           &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...},
+ 		ReadyReplicas:      0,
- 		ReadyReplicas:      3,
+ 		Replicas:           0,
- 		Replicas:           3,
- 		Replicas: &3,
+ 		Replicas: &4,
+ 		ResourceVersion:            "",
- 		ResourceVersion:            "1777559534987935023",
+ 				RestartPolicy:                 "",
- 				RestartPolicy:                 "Always",
- 		RevisionHistoryLimit:                 &10,
+ 		RevisionHistoryLimit:                 nil,
+ 				SchedulerName: "",
- 				SchedulerName: "default-scheduler",
  								SecretName:  "internal-some-name",
  								SecretName:  "some-name-mysql-init",
  								SecretName:  "some-name-ssl",
  								SecretName:  "some-name-ssl-internal",
  								SecretName:  "some-name-vault",
  							Secret: &v1.SecretVolumeSource{
  						SecurityContext:          nil,
  		Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},
  		SelfLink:                   "",
  				ServiceAccountName:            "default",
  		ServiceName:                          "some-name-pxc",
  				SetHostnameAsFQDN:         nil,
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
  				Spec: v1.PersistentVolumeClaimSpec{
  			Spec: v1.PodSpec{
  	Spec: v1.StatefulSetSpec{
  						StartupProbe:             nil,
  				Status: v1.PersistentVolumeClaimStatus{
  	Status: v1.StatefulSetStatus{
  					StorageClassName:          nil,
  				Subdomain:     "",
- 				Subresource: "status",
  							SuccessThreshold:              1,
  		Template: v1.PodTemplateSpec{
  				TerminationGracePeriodSeconds: &600,
  							TerminationGracePeriodSeconds: nil,
+ 						TerminationMessagePath:   "",
- 						TerminationMessagePath:   "/dev/termination-log",
+ 						TerminationMessagePolicy: "",
- 						TerminationMessagePolicy: "File",
  							TimeoutSeconds:                5,
- 				Time:       s"2026-04-30 14:28:13 +0000 UTC",
- 				Time:        s"2026-04-30 14:32:14 +0000 UTC",
  				Tolerations:   nil,
- 				TopologySpreadConstraints: nil,
+ 				TopologySpreadConstraints: []v1.TopologySpreadConstraint{},
  				TypeMeta:   {},
  	TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"},
+ 		UID:                        "",
- 		UID:                        "de670130-6696-426f-bc95-e97b33fc6976",
+ 		UpdatedReplicas:    0,
- 		UpdatedReplicas:    3,
+ 		UpdateRevision:     "",
- 		UpdateRevision:     "some-name-pxc-89f54f49d",
  		UpdateStrategy:                       {Type: "RollingUpdate", RollingUpdate: &{Partition: &0}},
  &v1.StatefulSet{
  					VolumeAttributesClassName: nil,
  		VolumeClaimTemplates: []v1.PersistentVolumeClaim{
  						VolumeDevices: nil,
- 					VolumeMode:                &"Filesystem",
+ 					VolumeMode:                nil,
  						VolumeMounts:  {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...},
  					VolumeName:                "",
  						VolumeSource: v1.VolumeSource{
  				Volumes: []v1.Volume{
  							VsphereVolume: nil,
  						WorkingDir: "",
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n sst-retry-limit-30476 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched (no change)
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.M43sdBNCMg
++ mktemp
+ local LAST_ERR=/tmp/tmp.iitvcymCYR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.M43sdBNCMg
perconaxtradbcluster.pxc.percona.com "some-name" deleted from sst-retry-limit-30476 namespace
+ cat /tmp/tmp.iitvcymCYR
+ rm /tmp/tmp.M43sdBNCMg /tmp/tmp.iitvcymCYR
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.06DRCqd5xR
++ mktemp
+ local LAST_ERR=/tmp/tmp.xUW7vYVjd1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.06DRCqd5xR
No resources found
+ cat /tmp/tmp.xUW7vYVjd1
+ rm /tmp/tmp.06DRCqd5xR /tmp/tmp.xUW7vYVjd1
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.sHh0TjNhLM
++ mktemp
+ local LAST_ERR=/tmp/tmp.LHXimiY5zR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.sHh0TjNhLM
No resources found
+ cat /tmp/tmp.LHXimiY5zR
+ rm /tmp/tmp.sHh0TjNhLM /tmp/tmp.LHXimiY5zR
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.B71bT3xAFM
++ mktemp
+ local LAST_ERR=/tmp/tmp.Atuc3jf8BV
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.B71bT3xAFM
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.Atuc3jf8BV
+ rm /tmp/tmp.B71bT3xAFM /tmp/tmp.Atuc3jf8BV
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace sst-retry-limit-30476
+ rm -rf /tmp/tmp.E0OCQMLtee
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
++ mktemp
+ desc 'test passed'
+ local LAST_OUT=/tmp/tmp.igL5mpwEzT
+ set +o xtrace
+ local LAST_OUT=/tmp/tmp.Rt18vK713R


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.qbi6t0S1aR
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.aFKWle5qL9
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace sst-retry-limit-30476
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator