Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/logs/sst-retry-limit-8-0.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + main + local cluster=some-name + local joiner_pod=some-name-pxc-3 + donor_candidates=('some-name-pxc-1' 'some-name-pxc-2') + local donor_candidates + create_infra sst-retry-limit-22208 + local ns=sst-retry-limit-22208 + '[' -n pxc-operator ']' + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + [[ -z sst-retry-limit-5278 ]] + [[ -z some-name ]] + kubectl_bin patch pxc -n sst-retry-limit-5278 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.kKD0C5yhYd ++ mktemp + local LAST_ERR=/tmp/tmp.0jNyoVFzfo + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc -n sst-retry-limit-5278 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kKD0C5yhYd perconaxtradbcluster.pxc.percona.com/some-name patched (no change) + cat /tmp/tmp.0jNyoVFzfo + rm /tmp/tmp.kKD0C5yhYd /tmp/tmp.0jNyoVFzfo + return 0 + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found perconaxtradbcluster.pxc.percona.com "some-name" deleted from sst-retry-limit-5278 namespace + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + kubectl_bin get ns + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.7KKpso8NXN ++ mktemp + local LAST_OUT=/tmp/tmp.YWeOj0NL5E ++ mktemp + local LAST_ERR=/tmp/tmp.gptOr9V12G ++ mktemp + local exit_status=0 + local LAST_ERR=/tmp/tmp.jpMrV5vBO4 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7KKpso8NXN + cat /tmp/tmp.jpMrV5vBO4 + rm /tmp/tmp.7KKpso8NXN /tmp/tmp.jpMrV5vBO4 + return 0 namespace "sst-retry-limit-5278" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YWeOj0NL5E namespace "pxc-operator" deleted + cat /tmp/tmp.gptOr9V12G + rm /tmp/tmp.YWeOj0NL5E /tmp/tmp.gptOr9V12G + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.rzXFrzyquI ++ mktemp + local LAST_ERR=/tmp/tmp.tDuoywSUqB + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rzXFrzyquI namespace/pxc-operator created + cat /tmp/tmp.tDuoywSUqB + rm /tmp/tmp.rzXFrzyquI /tmp/tmp.tDuoywSUqB + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.AlwKIfeHrv +++ mktemp ++ local LAST_ERR=/tmp/tmp.AY8D3Vat6L ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AlwKIfeHrv ++ cat /tmp/tmp.AY8D3Vat6L ++ rm /tmp/tmp.AlwKIfeHrv /tmp/tmp.AY8D3Vat6L ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2525-c81e47f0-4-cluster11 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.WLyzZ3YHlF ++ mktemp + local LAST_ERR=/tmp/tmp.icbdniLBoE + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2525-c81e47f0-4-cluster11 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WLyzZ3YHlF Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2525-c81e47f0-4-cluster11" modified. + cat /tmp/tmp.icbdniLBoE + rm /tmp/tmp.WLyzZ3YHlF /tmp/tmp.icbdniLBoE + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.chqOv6QHk5 ++ mktemp + local LAST_ERR=/tmp/tmp.aF0kADRqmS + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.chqOv6QHk5 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.aF0kADRqmS + rm /tmp/tmp.chqOv6QHk5 /tmp/tmp.aF0kADRqmS + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.fKvjqf454Q ++ mktemp + local LAST_ERR=/tmp/tmp.QMzAFrgJzJ + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fKvjqf454Q customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.QMzAFrgJzJ E0806 04:49:21.712066 1599101 reflector.go:227] "Failed to watch" err="Get \"https://34.60.104.255/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusters.pxc.percona.com&resourceVersion=1785991760307253000&timeoutSeconds=343&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.fKvjqf454Q /tmp/tmp.QMzAFrgJzJ + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.1uu76iWKxk ++ mktemp + local LAST_ERR=/tmp/tmp.IlvdQllpZk + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1uu76iWKxk customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.IlvdQllpZk E0806 04:49:24.198959 1599557 reflector.go:227] "Failed to watch" err="Get \"https://34.60.104.255/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterbackups.pxc.percona.com&resourceVersion=1785991760307253000&timeoutSeconds=438&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.1uu76iWKxk /tmp/tmp.IlvdQllpZk + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.6cK6mDqEXA ++ mktemp + local LAST_ERR=/tmp/tmp.yXA6O95GHo + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6cK6mDqEXA customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.yXA6O95GHo E0806 04:49:27.516897 1599919 reflector.go:227] "Failed to watch" err="Get \"https://34.60.104.255/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterrestores.pxc.percona.com&resourceVersion=1785991765307280000&timeoutSeconds=451&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.6cK6mDqEXA /tmp/tmp.yXA6O95GHo + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/deploy/cw-rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.9PZhWeEgL4 ++ mktemp + local LAST_ERR=/tmp/tmp.uvLp04N8V7 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9PZhWeEgL4 clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.uvLp04N8V7 + rm /tmp/tmp.9PZhWeEgL4 /tmp/tmp.uvLp04N8V7 + return 0 + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2525-c81e47f0^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.fx1GNr2xwx ++ mktemp + local LAST_ERR=/tmp/tmp.buBuKE6TWH + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fx1GNr2xwx deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.buBuKE6TWH + rm /tmp/tmp.fx1GNr2xwx /tmp/tmp.buBuKE6TWH + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.9CGUiTd4Oq ++ mktemp + local LAST_ERR=/tmp/tmp.WCDt1XLDHn + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9CGUiTd4Oq pod/percona-xtradb-cluster-operator-6699755d74-b52dk condition met + cat /tmp/tmp.WCDt1XLDHn E0806 04:49:49.364560 1602905 reflector.go:227] "Failed to watch" err="Get \"https://34.60.104.255/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-6699755d74-b52dk&resourceVersion=1785991789004891000&timeoutSeconds=569&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.9CGUiTd4Oq /tmp/tmp.WCDt1XLDHn + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.a4nhJiYMGH +++ mktemp ++ local LAST_ERR=/tmp/tmp.hIYbXowLd4 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.a4nhJiYMGH ++ cat /tmp/tmp.hIYbXowLd4 ++ rm /tmp/tmp.a4nhJiYMGH /tmp/tmp.hIYbXowLd4 ++ return 0 + wait_pod percona-xtradb-cluster-operator-6699755d74-b52dk 480 pxc-operator + local pod=percona-xtradb-cluster-operator-6699755d74-b52dk + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-6699755d74-b52dk ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-6699755d74-b52dk condition met E0806 04:49:56.566530 1603813 reflector.go:227] "Failed to watch" err="Get \"https://34.60.104.255/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-6699755d74-b52dk&resourceVersion=1785991793349734000&timeoutSeconds=503&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-6699755d74-b52dk to become Ready.Ok + sleep 3 + create_namespace sst-retry-limit-22208 + local namespace=sst-retry-limit-22208 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces sst-retry-limit-22208' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces sst-retry-limit-22208 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace sst-retry-limit-22208 + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.Bd3npfCfi1 + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.8DQKtXJNQS ++ mktemp + local LAST_ERR=/tmp/tmp.qikVEKsdmP + local exit_status=0 ++ mktemp + local LAST_ERR=/tmp/tmp.DBNqVM8Lmb + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace sst-retry-limit-22208 + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace sst-retry-limit-22208 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8DQKtXJNQS + cat /tmp/tmp.DBNqVM8Lmb + rm /tmp/tmp.8DQKtXJNQS /tmp/tmp.DBNqVM8Lmb + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace sst-retry-limit-22208 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.Bd3npfCfi1 + cat /tmp/tmp.qikVEKsdmP Error from server (NotFound): namespaces "sst-retry-limit-22208" not found + rm /tmp/tmp.Bd3npfCfi1 /tmp/tmp.qikVEKsdmP + return 1 + : + wait_for_delete namespace/sst-retry-limit-22208 + local res=namespace/sst-retry-limit-22208 + echo -n 'waiting for namespace/sst-retry-limit-22208 to be deleted' waiting for namespace/sst-retry-limit-22208 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "sst-retry-limit-22208" not found + desc 'create namespace sst-retry-limit-22208' + set +o xtrace ----------------------------------------------------------------------------------- create namespace sst-retry-limit-22208 ----------------------------------------------------------------------------------- + kubectl_bin create namespace sst-retry-limit-22208 ++ mktemp + local LAST_OUT=/tmp/tmp.5U7o4ybUiJ ++ mktemp + local LAST_ERR=/tmp/tmp.ilJ2q7OPGh + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace sst-retry-limit-22208 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5U7o4ybUiJ namespace/sst-retry-limit-22208 created + cat /tmp/tmp.ilJ2q7OPGh + rm /tmp/tmp.5U7o4ybUiJ /tmp/tmp.ilJ2q7OPGh + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.PwQiqjIZ1W +++ mktemp ++ local LAST_ERR=/tmp/tmp.uhsWLCLlex ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PwQiqjIZ1W ++ cat /tmp/tmp.uhsWLCLlex ++ rm /tmp/tmp.PwQiqjIZ1W /tmp/tmp.uhsWLCLlex ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2525-c81e47f0-4-cluster11 --namespace=sst-retry-limit-22208 ++ mktemp + local LAST_OUT=/tmp/tmp.LbnFVrmqBr ++ mktemp + local LAST_ERR=/tmp/tmp.BtlECSwqYz + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2525-c81e47f0-4-cluster11 --namespace=sst-retry-limit-22208 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LbnFVrmqBr Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2525-c81e47f0-4-cluster11" modified. + cat /tmp/tmp.BtlECSwqYz + rm /tmp/tmp.LbnFVrmqBr /tmp/tmp.BtlECSwqYz + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.cA8vMOez4H ++ mktemp + local LAST_ERR=/tmp/tmp.qart66EPmD + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cA8vMOez4H secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.qart66EPmD + rm /tmp/tmp.cA8vMOez4H /tmp/tmp.qart66EPmD + return 0 + desc 'start 3-node cluster with SST retry limit' + set +o xtrace ----------------------------------------------------------------------------------- start 3-node cluster with SST retry limit ----------------------------------------------------------------------------------- + spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/conf/some-name.yml 3 10 /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/secrets.yml /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/client.yml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/conf/some-name.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.BmtwzsZlST ++ mktemp + local LAST_ERR=/tmp/tmp.O0dWFp5kg4 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BmtwzsZlST secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.O0dWFp5kg4 + rm /tmp/tmp.BmtwzsZlST /tmp/tmp.O0dWFp5kg4 + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2525-c81e47f0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/conf/client.yml + /usr/bin/sed -e s~minio-service.#namespace~minio-service.sst-retry-limit-22208~ ++ mktemp + local LAST_OUT=/tmp/tmp.fBp6HbuioI ++ mktemp + local LAST_ERR=/tmp/tmp.R7oMjnT8IB + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fBp6HbuioI deployment.apps/pxc-client created + cat /tmp/tmp.R7oMjnT8IB + rm /tmp/tmp.fBp6HbuioI /tmp/tmp.R7oMjnT8IB + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/conf/some-name.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/conf/some-name.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/conf/some-name.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/conf/some-name.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2525-c81e47f0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.sst-retry-limit-22208~ + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/conf/some-name.yml + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_OUT=/tmp/tmp.L4R2tPMLEv + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + local LAST_ERR=/tmp/tmp.3NDkoMnyp6 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.L4R2tPMLEv perconaxtradbcluster.pxc.percona.com/some-name created + cat /tmp/tmp.3NDkoMnyp6 + rm /tmp/tmp.L4R2tPMLEv /tmp/tmp.3NDkoMnyp6 + return 0 + desc 'waiting pods to be running for some-name' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for some-name ----------------------------------------------------------------------------------- ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vUvy76u0CW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4jmfGmGBXE +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.vUvy76u0CW +++ cat /tmp/tmp.4jmfGmGBXE +++ rm /tmp/tmp.vUvy76u0CW /tmp/tmp.4jmfGmGBXE +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NNgtkzVTFV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.aQka8hsHC3 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.NNgtkzVTFV +++ cat /tmp/tmp.aQka8hsHC3 +++ rm /tmp/tmp.NNgtkzVTFV /tmp/tmp.aQka8hsHC3 +++ return 0 ++ [[ false == \t\r\u\e ]] + local proxy= ++ get_proxy_size some-name ++ local cluster=some-name ++ local size=0 +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.43Y5oBioz7 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mtivyQJrt5 +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.43Y5oBioz7 +++ cat /tmp/tmp.mtivyQJrt5 +++ rm /tmp/tmp.43Y5oBioz7 /tmp/tmp.mtivyQJrt5 +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6MWpFVVYtt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gHadj5Iv2G +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.6MWpFVVYtt +++ cat /tmp/tmp.gHadj5Iv2G +++ rm /tmp/tmp.6MWpFVVYtt /tmp/tmp.gHadj5Iv2G +++ return 0 ++ [[ false == \t\r\u\e ]] ++ echo 0 + local proxy_size=0 + wait_for_running some-name-pxc 3 + local name=some-name-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z some-name-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for some-name-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for some-name-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in $(seq 0 $last_pod) + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo some-name-pxc-0 + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met waiting for pod/some-name-pxc-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod some-name-pxc-1 480 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ echo some-name-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod some-name-pxc-2 480 + local pod=some-name-pxc-2 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo some-name-pxc-2 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-2 condition met waiting for pod/some-name-pxc-2 to become Ready.Ok + wait_for_running '' 0 + local name= + local size=0 + local last_pod + local max_retry=480 + [[ -z '' ]] + desc 'No resource provided to wait for running pods' + set +o xtrace ----------------------------------------------------------------------------------- No resource provided to wait for running pods ----------------------------------------------------------------------------------- + return + wait_cluster_consistency some-name 3 0 + local cluster_name=some-name + local pxc_size=3 + local proxy_size=0 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/some-name to be ready' waiting for pxc/some-name to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TUhPHJWOL3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.pjZBvZDa5H ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TUhPHJWOL3 ++ cat /tmp/tmp.pjZBvZDa5H ++ rm /tmp/tmp.TUhPHJWOL3 /tmp/tmp.pjZBvZDa5H ++ return 0 + state=ready ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RMramy6EtS +++ mktemp ++ local LAST_ERR=/tmp/tmp.aaoJNaslCl ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RMramy6EtS ++ cat /tmp/tmp.aaoJNaslCl ++ rm /tmp/tmp.RMramy6EtS /tmp/tmp.aaoJNaslCl ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 0 -gt 0 ]] + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 0 == \0 ]] + echo + return + sleep 10 ++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.UMDaNdz2fi +++ mktemp ++ local LAST_ERR=/tmp/tmp.MHZe5JY3hU ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UMDaNdz2fi ++ cat /tmp/tmp.MHZe5JY3hU ++ rm /tmp/tmp.UMDaNdz2fi /tmp/tmp.MHZe5JY3hU ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0dhx6edE6I +++ mktemp ++ local LAST_ERR=/tmp/tmp.Vekq7lUwtW ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0dhx6edE6I ++ cat /tmp/tmp.Vekq7lUwtW ++ rm /tmp/tmp.0dhx6edE6I /tmp/tmp.Vekq7lUwtW ++ return 0 + client_pod=pxc-client-67fc4995bb-rbwbm + wait_pod pxc-client-67fc4995bb-rbwbm + local pod=pxc-client-67fc4995bb-rbwbm + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-rbwbm ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-rbwbm condition met E0806 04:55:29.162927 1646389 reflector.go:227] "Failed to watch" err="Get \"https://34.60.104.255/api/v1/namespaces/sst-retry-limit-22208/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-rbwbm&resourceVersion=1785992127466531000&timeoutSeconds=516&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-rbwbm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data some-name 100500 no-proxy -P3306 + local cluster=some-name + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password some-name root ++ local cluster=some-name ++ local user=root +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8hnPtnmLnX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Lygfeev5SY +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.8hnPtnmLnX +++ cat /tmp/tmp.Lygfeev5SY +++ rm /tmp/tmp.8hnPtnmLnX /tmp/tmp.Lygfeev5SY +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.C7EeTwflRb +++ mktemp ++ local LAST_ERR=/tmp/tmp.8EsSNKadyp ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.C7EeTwflRb ++ cat /tmp/tmp.8EsSNKadyp ++ rm /tmp/tmp.C7EeTwflRb /tmp/tmp.8EsSNKadyp ++ return 0 + root_pass=root_password ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BAgWkBjygT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hRlV3ohcQm +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.BAgWkBjygT +++ cat /tmp/tmp.hRlV3ohcQm +++ rm /tmp/tmp.BAgWkBjygT /tmp/tmp.hRlV3ohcQm +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7KgZkWk3HC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tmWn9MJ5Ms +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.7KgZkWk3HC +++ cat /tmp/tmp.tmWn9MJ5Ms +++ rm /tmp/tmp.7KgZkWk3HC /tmp/tmp.tmWn9MJ5Ms +++ return 0 ++ [[ false == \t\r\u\e ]] + proxy= + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h some-name-pxc-0.some-name-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Zo0ELGiwoH +++ mktemp ++ local LAST_ERR=/tmp/tmp.l2h9cvIM1B ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Zo0ELGiwoH ++ cat /tmp/tmp.l2h9cvIM1B ++ rm /tmp/tmp.Zo0ELGiwoH /tmp/tmp.l2h9cvIM1B ++ return 0 + client_pod=pxc-client-67fc4995bb-rbwbm + wait_pod pxc-client-67fc4995bb-rbwbm + local pod=pxc-client-67fc4995bb-rbwbm + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-67fc4995bb-rbwbm ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-rbwbm condition met E0806 04:55:38.710960 1647586 reflector.go:227] "Failed to watch" err="Get \"https://34.60.104.255/api/v1/namespaces/sst-retry-limit-22208/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-rbwbm&resourceVersion=1785992137892348000&timeoutSeconds=431&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-rbwbm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data some-name select-1 no-proxy -P3306 + local cluster=some-name + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password some-name root ++ local cluster=some-name ++ local user=root +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.boWLLVbqGe ++++ mktemp +++ local LAST_ERR=/tmp/tmp.87ju93Y6An +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.boWLLVbqGe +++ cat /tmp/tmp.87ju93Y6An +++ rm /tmp/tmp.boWLLVbqGe /tmp/tmp.87ju93Y6An +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.FctSou96xE +++ mktemp ++ local LAST_ERR=/tmp/tmp.PnFjXfiYWv ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FctSou96xE ++ cat /tmp/tmp.PnFjXfiYWv ++ rm /tmp/tmp.FctSou96xE /tmp/tmp.PnFjXfiYWv ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size some-name ++ local cluster=some-name ++ local size=0 +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.n3Jm3LgEXy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ogkNDCJnMg +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.n3Jm3LgEXy +++ cat /tmp/tmp.ogkNDCJnMg +++ rm /tmp/tmp.n3Jm3LgEXy /tmp/tmp.ogkNDCJnMg +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rKJeqjemoV +++ mktemp ++ local LAST_ERR=/tmp/tmp.ujvpP2GGVo ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rKJeqjemoV ++ cat /tmp/tmp.ujvpP2GGVo ++ rm /tmp/tmp.rKJeqjemoV /tmp/tmp.ujvpP2GGVo ++ return 0 + client_pod=pxc-client-67fc4995bb-rbwbm + wait_pod pxc-client-67fc4995bb-rbwbm + local pod=pxc-client-67fc4995bb-rbwbm + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-rbwbm ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-rbwbm condition met E0806 04:56:16.259503 1650991 reflector.go:227] "Failed to watch" err="Get \"https://34.60.104.255/api/v1/namespaces/sst-retry-limit-22208/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-rbwbm&resourceVersion=1785992175485757000&timeoutSeconds=399&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-rbwbm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xcRCCWmxWa/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xcRCCWmxWa/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.xcRCCWmxWa/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-1.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h some-name-pxc-1.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-1.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h some-name-pxc-1.some-name-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5wWxFKNd1z +++ mktemp ++ local LAST_ERR=/tmp/tmp.nr3rwTHmoi ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5wWxFKNd1z ++ cat /tmp/tmp.nr3rwTHmoi ++ rm /tmp/tmp.5wWxFKNd1z /tmp/tmp.nr3rwTHmoi ++ return 0 + client_pod=pxc-client-67fc4995bb-rbwbm + wait_pod pxc-client-67fc4995bb-rbwbm + local pod=pxc-client-67fc4995bb-rbwbm + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-rbwbm ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-rbwbm condition met E0806 04:56:22.319311 1651469 reflector.go:227] "Failed to watch" err="Get \"https://34.60.104.255/api/v1/namespaces/sst-retry-limit-22208/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-rbwbm&resourceVersion=1785992180776405000&timeoutSeconds=541&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-rbwbm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xcRCCWmxWa/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xcRCCWmxWa/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.xcRCCWmxWa/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-2.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h some-name-pxc-2.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h some-name-pxc-2.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h some-name-pxc-2.some-name-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RZcInAKrbx +++ mktemp ++ local LAST_ERR=/tmp/tmp.9ontctBszS ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RZcInAKrbx ++ cat /tmp/tmp.9ontctBszS ++ rm /tmp/tmp.RZcInAKrbx /tmp/tmp.9ontctBszS ++ return 0 + client_pod=pxc-client-67fc4995bb-rbwbm + wait_pod pxc-client-67fc4995bb-rbwbm + local pod=pxc-client-67fc4995bb-rbwbm + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-rbwbm ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-rbwbm condition met E0806 04:56:27.940423 1652064 reflector.go:227] "Failed to watch" err="Get \"https://34.60.104.255/api/v1/namespaces/sst-retry-limit-22208/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-rbwbm&resourceVersion=1785992186607131000&timeoutSeconds=402&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-rbwbm to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xcRCCWmxWa/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xcRCCWmxWa/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2525/e2e-tests/sst-retry-limit/compare/select-1.sql /tmp/tmp.xcRCCWmxWa/select-1.sql + is_keyring_plugin_in_use some-name + local cluster=some-name + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + kubectl exec some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + grep -E -o 'early-plugin-load=keyring_\w+.so' + return 1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.secretsName}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.D5nriqxBWd +++ mktemp ++ local LAST_ERR=/tmp/tmp.HIOfvAhhYa ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.D5nriqxBWd ++ cat /tmp/tmp.HIOfvAhhYa ++ rm /tmp/tmp.D5nriqxBWd /tmp/tmp.HIOfvAhhYa ++ return 0 + secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.mRCCfipb42 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZiRgaiyB0x ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mRCCfipb42 ++ cat /tmp/tmp.ZiRgaiyB0x ++ rm /tmp/tmp.mRCCfipb42 /tmp/tmp.ZiRgaiyB0x ++ return 0 + root_pass=root_password + desc 'scale to 4 nodes to trigger SST' + set +o xtrace ----------------------------------------------------------------------------------- scale to 4 nodes to trigger SST ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type=merge -p '{"spec":{"pxc":{"size":4}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.jTJqRSvKcS ++ mktemp + local LAST_ERR=/tmp/tmp.J7WrErgQ7o + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch pxc some-name --type=merge -p '{"spec":{"pxc":{"size":4}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jTJqRSvKcS perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.J7WrErgQ7o + rm /tmp/tmp.jTJqRSvKcS /tmp/tmp.J7WrErgQ7o + return 0 + desc 'verify the joiner pod has SST retry support wired in' + set +o xtrace ----------------------------------------------------------------------------------- verify the joiner pod has SST retry support wired in ----------------------------------------------------------------------------------- + local retry + retry=0 + echo -n 'waiting for pod/some-name-pxc-3 container/pxc to start' waiting for pod/some-name-pxc-3 container/pxc to start+ local joiner_container_started ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8vQv6S5s0M +++ mktemp ++ local LAST_ERR=/tmp/tmp.OZZLalsUVV ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8vQv6S5s0M ++ cat /tmp/tmp.OZZLalsUVV ++ rm /tmp/tmp.8vQv6S5s0M /tmp/tmp.OZZLalsUVV ++ return 0 + joiner_container_started= + [[ '' == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.m0Lywk41Da +++ mktemp ++ local LAST_ERR=/tmp/tmp.aCm2X0e3ze ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.m0Lywk41Da ++ cat /tmp/tmp.aCm2X0e3ze ++ rm /tmp/tmp.m0Lywk41Da /tmp/tmp.aCm2X0e3ze ++ return 0 + joiner_container_started= + [[ '' == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d75G7l9RKg +++ mktemp ++ local LAST_ERR=/tmp/tmp.MlFyqGeOYX ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.d75G7l9RKg ++ cat /tmp/tmp.MlFyqGeOYX ++ rm /tmp/tmp.d75G7l9RKg /tmp/tmp.MlFyqGeOYX ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XJbwWcXCmT +++ mktemp ++ local LAST_ERR=/tmp/tmp.CMVYjuV3Pq ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XJbwWcXCmT ++ cat /tmp/tmp.CMVYjuV3Pq ++ rm /tmp/tmp.XJbwWcXCmT /tmp/tmp.CMVYjuV3Pq ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OFzv50u2tB +++ mktemp ++ local LAST_ERR=/tmp/tmp.DTk0nj4Na1 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OFzv50u2tB ++ cat /tmp/tmp.DTk0nj4Na1 ++ rm /tmp/tmp.OFzv50u2tB /tmp/tmp.DTk0nj4Na1 ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d3Dvf76Uw9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.IrwamRkGUu ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.d3Dvf76Uw9 ++ cat /tmp/tmp.IrwamRkGUu ++ rm /tmp/tmp.d3Dvf76Uw9 /tmp/tmp.IrwamRkGUu ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xEP9Z0GqcO +++ mktemp ++ local LAST_ERR=/tmp/tmp.ssVD2y6Te2 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xEP9Z0GqcO ++ cat /tmp/tmp.ssVD2y6Te2 ++ rm /tmp/tmp.xEP9Z0GqcO /tmp/tmp.ssVD2y6Te2 ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=7 + [[ 7 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hdIrBiOnnB +++ mktemp ++ local LAST_ERR=/tmp/tmp.8eeZlqcs8s ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hdIrBiOnnB ++ cat /tmp/tmp.8eeZlqcs8s ++ rm /tmp/tmp.hdIrBiOnnB /tmp/tmp.8eeZlqcs8s ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=8 + [[ 8 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RWUGGnKRmo +++ mktemp ++ local LAST_ERR=/tmp/tmp.aG10BH79Xz ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RWUGGnKRmo ++ cat /tmp/tmp.aG10BH79Xz ++ rm /tmp/tmp.RWUGGnKRmo /tmp/tmp.aG10BH79Xz ++ return 0 + joiner_container_started=false + [[ false == \t\r\u\e ]] + sleep 1 + echo -n . .+ retry=9 + [[ 9 -ge 240 ]] ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[]? | select(.name == "pxc") | .started // false' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tD2QiQxR6I +++ mktemp ++ local LAST_ERR=/tmp/tmp.DAAVZVBE3L ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tD2QiQxR6I ++ cat /tmp/tmp.DAAVZVBE3L ++ rm /tmp/tmp.tD2QiQxR6I /tmp/tmp.DAAVZVBE3L ++ return 0 + joiner_container_started=true + [[ true == \t\r\u\e ]] + echo + desc 'interrupt SST twice so the joiner exhausts allowed retries' + set +o xtrace ----------------------------------------------------------------------------------- interrupt SST twice so the joiner exhausts allowed retries ----------------------------------------------------------------------------------- + interrupt_donor_during_sst root_password some-name-pxc-3 0 some-name-pxc-1 some-name-pxc-2 + local root_pass=root_password + local joiner_pod=some-name-pxc-3 + local current_restart_count=0 + local target_restart_count=1 + local donor_pod + donor_candidates=('some-name-pxc-1' 'some-name-pxc-2') + local donor_candidates + local retry=0 + local observed_restart_count + echo -n 'waiting for pod/some-name-pxc-3 restartCount=0 SST attempt' waiting for pod/some-name-pxc-3 restartCount=0 SST attempt+ : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=0 + [[ 0 == \0 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + kubectl_bin logs some-name-pxc-3 -c pxc + grep -q 'Waiting for SST streaming to complete!' + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] + : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=0 + [[ 0 == \0 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + echo + break ++ detect_active_donor root_password 120 some-name-pxc-1 some-name-pxc-2 ++ local root_pass=root_password ++ shift ++ local max_retry=120 ++ shift ++ local retry=0 ++ local donor_pod ++ local donor_rejects_queries ++ local local_state ++ echo -n 'waiting to detect active donor' waiting to detect active donor++ : ++ for donor_pod in "$@" +++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace +++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}' ++ donor_rejects_queries=OFF +++ awk '$1 == "wsrep_local_state_comment" {print $2}' +++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace ++ local_state=Donor/Desynced ++ [[ OFF == \O\N ]] ++ [[ Donor/Desynced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]] ++ echo some-name-pxc-1 ++ return 0 + donor_pod=some-name-pxc-1 + echo 'interrupting donor pod/some-name-pxc-1 during SST attempt 1' interrupting donor pod/some-name-pxc-1 during SST attempt 1 + kubectl_bin delete pod some-name-pxc-1 --wait=false --grace-period=0 --force ++ mktemp + local LAST_OUT=/tmp/tmp.FIvv9luird ++ mktemp + local LAST_ERR=/tmp/tmp.UUfyuL2YpV + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pod some-name-pxc-1 --wait=false --grace-period=0 --force + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FIvv9luird pod "some-name-pxc-1" force deleted from sst-retry-limit-22208 namespace + cat /tmp/tmp.UUfyuL2YpV Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.FIvv9luird /tmp/tmp.UUfyuL2YpV + return 0 + wait_for_pxc_restart_count some-name-pxc-3 1 600 + local pod=some-name-pxc-3 + local expected=1 + local max_retry=600 + local retry=0 + echo -n 'waiting for pod/some-name-pxc-3 restartCount to reach 1' waiting for pod/some-name-pxc-3 restartCount to reach 1++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=7 + [[ 7 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=8 + [[ 8 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=9 + [[ 9 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=10 + [[ 10 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=11 + [[ 11 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=12 + [[ 12 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=13 + [[ 13 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=14 + [[ 14 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=15 + [[ 15 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=16 + [[ 16 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=17 + [[ 17 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=18 + [[ 18 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=19 + [[ 19 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=20 + [[ 20 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=21 + [[ 21 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=22 + [[ 22 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=23 + [[ 23 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=24 + [[ 24 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=25 + [[ 25 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=26 + [[ 26 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=27 + [[ 27 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=28 + [[ 28 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=29 + [[ 29 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=30 + [[ 30 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=31 + [[ 31 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=32 + [[ 32 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=33 + [[ 33 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=34 + [[ 34 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=35 + [[ 35 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=36 + [[ 36 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=37 + [[ 37 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=38 + [[ 38 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=39 + [[ 39 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=40 + [[ 40 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=41 + [[ 41 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=42 + [[ 42 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=43 + [[ 43 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=44 + [[ 44 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=45 + [[ 45 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=46 + [[ 46 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=47 + [[ 47 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=48 + [[ 48 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=49 + [[ 49 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=50 + [[ 50 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 0 -ge 1 ]] + sleep 1 + echo -n . .+ retry=51 + [[ 51 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 1 ]] + echo + wait_pod some-name-pxc-1 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo some-name-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met E0806 04:59:21.511495 1670131 reflector.go:227] "Failed to watch" err="Get \"https://34.60.104.255/api/v1/namespaces/sst-retry-limit-22208/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-pxc-1&resourceVersion=1785992357876479012&timeoutSeconds=465&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/some-name-pxc-1 to become Ready.Ok + interrupt_donor_during_sst root_password some-name-pxc-3 1 some-name-pxc-1 some-name-pxc-2 + local root_pass=root_password + local joiner_pod=some-name-pxc-3 + local current_restart_count=1 + local target_restart_count=2 + local donor_pod + donor_candidates=('some-name-pxc-1' 'some-name-pxc-2') + local donor_candidates + local retry=0 + local observed_restart_count + echo -n 'waiting for pod/some-name-pxc-3 restartCount=1 SST attempt' waiting for pod/some-name-pxc-3 restartCount=1 SST attempt+ : ++ get_pxc_restart_count some-name-pxc-3 + observed_restart_count=1 + [[ 1 == \1 ]] + kubectl_bin exec some-name-pxc-3 -c pxc -- test -f /var/lib/mysql/sst_in_progress + echo + break ++ detect_active_donor root_password 120 some-name-pxc-1 some-name-pxc-2 ++ local root_pass=root_password ++ shift ++ local max_retry=120 ++ shift ++ local retry=0 ++ local donor_pod ++ local donor_rejects_queries ++ local local_state ++ echo -n 'waiting to detect active donor' waiting to detect active donor++ : ++ for donor_pod in "$@" +++ awk '$1 == "wsrep_sst_donor_rejects_queries" {print $2}' +++ run_mysql_local 'SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL VARIABLES LIKE '\''wsrep_sst_donor_rejects_queries'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace ++ donor_rejects_queries=OFF +++ awk '$1 == "wsrep_local_state_comment" {print $2}' +++ run_mysql_local 'SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 +++ local 'command=SHOW GLOBAL STATUS LIKE '\''wsrep_local_state_comment'\'';' +++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' +++ local pod=some-name-pxc-1 +++ local container_name= +++ set +o xtrace ++ local_state=Donor/Desynced ++ [[ OFF == \O\N ]] ++ [[ Donor/Desynced == \D\o\n\o\r\/\D\e\s\y\n\c\e\d ]] ++ echo some-name-pxc-1 ++ return 0 + donor_pod=some-name-pxc-1 + echo 'interrupting donor pod/some-name-pxc-1 during SST attempt 2' interrupting donor pod/some-name-pxc-1 during SST attempt 2 + kubectl_bin delete pod some-name-pxc-1 --wait=false --grace-period=0 --force ++ mktemp + local LAST_OUT=/tmp/tmp.OAvQTvCIU5 ++ mktemp + local LAST_ERR=/tmp/tmp.oBOsto4ilk + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pod some-name-pxc-1 --wait=false --grace-period=0 --force + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OAvQTvCIU5 pod "some-name-pxc-1" force deleted from sst-retry-limit-22208 namespace + cat /tmp/tmp.oBOsto4ilk Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.OAvQTvCIU5 /tmp/tmp.oBOsto4ilk + return 0 + wait_for_pxc_restart_count some-name-pxc-3 2 600 + local pod=some-name-pxc-3 + local expected=2 + local max_retry=600 + local retry=0 + echo -n 'waiting for pod/some-name-pxc-3 restartCount to reach 2' waiting for pod/some-name-pxc-3 restartCount to reach 2++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=1 + [[ 1 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=2 + [[ 2 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=3 + [[ 3 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=4 + [[ 4 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=5 + [[ 5 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=6 + [[ 6 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=7 + [[ 7 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=8 + [[ 8 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=9 + [[ 9 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=10 + [[ 10 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=11 + [[ 11 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=12 + [[ 12 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=13 + [[ 13 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=14 + [[ 14 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=15 + [[ 15 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=16 + [[ 16 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=17 + [[ 17 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=18 + [[ 18 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=19 + [[ 19 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=20 + [[ 20 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=21 + [[ 21 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=22 + [[ 22 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=23 + [[ 23 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=24 + [[ 24 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=25 + [[ 25 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=26 + [[ 26 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=27 + [[ 27 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=28 + [[ 28 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=29 + [[ 29 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=30 + [[ 30 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=31 + [[ 31 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=32 + [[ 32 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=33 + [[ 33 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=34 + [[ 34 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=35 + [[ 35 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 1 -ge 2 ]] + sleep 1 + echo -n . .+ retry=36 + [[ 36 -ge 600 ]] ++ get_pxc_restart_count some-name-pxc-3 + [[ 2 -ge 2 ]] + echo + wait_pod some-name-pxc-1 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo some-name-pxc-1 ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met E0806 05:02:00.759020 1690515 reflector.go:227] "Failed to watch" err="Get \"https://34.60.104.255/api/v1/namespaces/sst-retry-limit-22208/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-pxc-1&resourceVersion=1785992516156959012&timeoutSeconds=371&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/some-name-pxc-1 to become Ready.Ok + desc 'verify the joiner stops restarting after the retry limit is reached' + set +o xtrace ----------------------------------------------------------------------------------- verify the joiner stops restarting after the retry limit is reached ----------------------------------------------------------------------------------- + retry=0 + echo -n 'waiting for pod/some-name-pxc-3 logs to contain '\''SST retry limit reached.'\''' waiting for pod/some-name-pxc-3 logs to contain 'SST retry limit reached.'+ kubectl_bin logs some-name-pxc-3 -c pxc + grep -q 'SST retry limit reached.' + echo + desc 'verify retries stop after the limit' + set +o xtrace ----------------------------------------------------------------------------------- verify retries stop after the limit ----------------------------------------------------------------------------------- + local joiner_phase ++ kubectl_bin get pod some-name-pxc-3 -o 'jsonpath={.status.phase}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3g2kvA2Mrc +++ mktemp ++ local LAST_ERR=/tmp/tmp.6Cd1UqUFjo ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o 'jsonpath={.status.phase}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3g2kvA2Mrc ++ cat /tmp/tmp.6Cd1UqUFjo ++ rm /tmp/tmp.3g2kvA2Mrc /tmp/tmp.6Cd1UqUFjo ++ return 0 + joiner_phase=Running + local joiner_ready ++ jq -r '.status.conditions[] | select(.type == "Ready") | .status' ++ kubectl_bin get pod some-name-pxc-3 -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.fBwjOxyC5b +++ mktemp ++ local LAST_ERR=/tmp/tmp.AHtainVTMR ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fBwjOxyC5b ++ cat /tmp/tmp.AHtainVTMR ++ rm /tmp/tmp.fBwjOxyC5b /tmp/tmp.AHtainVTMR ++ return 0 + joiner_ready=False + [[ Running != \R\u\n\n\i\n\g ]] + [[ False != \F\a\l\s\e ]] + local restart_count_before ++ get_pxc_restart_count some-name-pxc-3 ++ local pod=some-name-pxc-3 ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[] | select(.name == "pxc") | .restartCount' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MYtX2HDR2t +++ mktemp ++ local LAST_ERR=/tmp/tmp.ezMhcObLBz ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MYtX2HDR2t ++ cat /tmp/tmp.ezMhcObLBz ++ rm /tmp/tmp.MYtX2HDR2t /tmp/tmp.ezMhcObLBz ++ return 0 + restart_count_before=2 + sleep_with_log 120 + local d=120 + log 'sleeping for 120 seconds' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-08-06T05:02:21+0000]' sleeping for 120 seconds [2026-08-06T05:02:21+0000] sleeping for 120 seconds + sleep 120 + local restart_count_after ++ get_pxc_restart_count some-name-pxc-3 ++ local pod=some-name-pxc-3 ++ kubectl_bin get pod some-name-pxc-3 -o json ++ jq -r '.status.containerStatuses[] | select(.name == "pxc") | .restartCount' +++ mktemp ++ local LAST_OUT=/tmp/tmp.i7hzvqMWZl +++ mktemp ++ local LAST_ERR=/tmp/tmp.EBJdg381mo ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pod some-name-pxc-3 -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.i7hzvqMWZl ++ cat /tmp/tmp.EBJdg381mo ++ rm /tmp/tmp.i7hzvqMWZl /tmp/tmp.EBJdg381mo ++ return 0 + restart_count_after=2 + [[ 2 != \2 ]] + desc 'verify donor candidates remain healthy' + set +o xtrace ----------------------------------------------------------------------------------- verify donor candidates remain healthy ----------------------------------------------------------------------------------- + local donor_result + local donor_pod + for donor_pod in "${donor_candidates[@]}" ++ run_mysql_local 'SELECT COUNT(*) FROM myApp.myApp;' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-1 ++ local 'command=SELECT COUNT(*) FROM myApp.myApp;' ++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' ++ local pod=some-name-pxc-1 ++ local container_name= ++ set +o xtrace + donor_result=1 + [[ 1 != \1 ]] + for donor_pod in "${donor_candidates[@]}" ++ run_mysql_local 'SELECT COUNT(*) FROM myApp.myApp;' '-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' some-name-pxc-2 ++ local 'command=SELECT COUNT(*) FROM myApp.myApp;' ++ local 'uri=-h127.0.0.1 -P3306 -uroot -p'\''root_password'\''' ++ local pod=some-name-pxc-2 ++ local container_name= ++ set +o xtrace + donor_result=1 + [[ 1 != \1 ]] + desc 'manually recover the joiner and verify the cluster becomes ready' + set +o xtrace ----------------------------------------------------------------------------------- manually recover the joiner and verify the cluster becomes ready ----------------------------------------------------------------------------------- + recover_joiner_after_sst_retry_limit some-name-pxc-3 + local joiner_pod=some-name-pxc-3 + echo 'removing SST retry marker files from pod/some-name-pxc-3' removing SST retry marker files from pod/some-name-pxc-3 + kubectl_bin exec some-name-pxc-3 -c pxc -- rm -f /var/lib/mysql/sst_retry_limit_reached /var/lib/mysql/sst_retry_count ++ mktemp + local LAST_OUT=/tmp/tmp.JKot1E9lW6 ++ mktemp + local LAST_ERR=/tmp/tmp.ud6L6kJIAA + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-pxc-3 -c pxc -- rm -f /var/lib/mysql/sst_retry_limit_reached /var/lib/mysql/sst_retry_count + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JKot1E9lW6 + cat /tmp/tmp.ud6L6kJIAA + rm /tmp/tmp.JKot1E9lW6 /tmp/tmp.ud6L6kJIAA + return 0 + wait_for_cluster_ready some-name 4 900 + local cluster=some-name + local pxc_size=4 + local max_retry=900 + local retry=0 + echo -n 'waiting for pxc/some-name to become ready with 4 PXC pods' waiting for pxc/some-name to become ready with 4 PXC pods++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ initializing == \r\e\a\d\y ]] + sleep 5 + echo -n . .+ retry=5 + [[ 5 -ge 900 ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ initializing == \r\e\a\d\y ]] + sleep 5 + echo -n . .+ retry=10 + [[ 10 -ge 900 ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ initializing == \r\e\a\d\y ]] + sleep 5 + echo -n . .+ retry=15 + [[ 15 -ge 900 ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ initializing == \r\e\a\d\y ]] + sleep 5 + echo -n . .+ retry=20 + [[ 20 -ge 900 ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ initializing == \r\e\a\d\y ]] + sleep 5 + echo -n . .+ retry=25 + [[ 25 -ge 900 ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ initializing == \r\e\a\d\y ]] + sleep 5 + echo -n . .+ retry=30 + [[ 30 -ge 900 ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ initializing == \r\e\a\d\y ]] + sleep 5 + echo -n . .+ retry=35 + [[ 35 -ge 900 ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ initializing == \r\e\a\d\y ]] + sleep 5 + echo -n . .+ retry=40 + [[ 40 -ge 900 ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ initializing == \r\e\a\d\y ]] + sleep 5 + echo -n . .+ retry=45 + [[ 45 -ge 900 ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ initializing == \r\e\a\d\y ]] + sleep 5 + echo -n . .+ retry=50 + [[ 50 -ge 900 ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ initializing == \r\e\a\d\y ]] + sleep 5 + echo -n . .+ retry=55 + [[ 55 -ge 900 ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ initializing == \r\e\a\d\y ]] + sleep 5 + echo -n . .+ retry=60 + [[ 60 -ge 900 ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ initializing == \r\e\a\d\y ]] + sleep 5 + echo -n . .+ retry=65 + [[ 65 -ge 900 ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' + [[ 4 == \4 ]] + echo + destroy sst-retry-limit-22208 + local namespace=sst-retry-limit-22208 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v 'the object has been modified' + tee /tmp/tmp.xcRCCWmxWa/operator.log + grep -v 'get backup status: Job.batch' + grep -v level=info + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + sort -u ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.otiHrPMXyQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.HDpHgHsl06 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.otiHrPMXyQ ++ cat /tmp/tmp.HDpHgHsl06 ++ rm /tmp/tmp.otiHrPMXyQ /tmp/tmp.HDpHgHsl06 ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-6699755d74-b52dk ++ mktemp + local LAST_OUT=/tmp/tmp.zcrxBqed7E ++ mktemp + local LAST_ERR=/tmp/tmp.fwwETUzeTG + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-6699755d74-b52dk + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zcrxBqed7E + cat /tmp/tmp.fwwETUzeTG + rm /tmp/tmp.zcrxBqed7E /tmp/tmp.fwwETUzeTG + return 0 ContainerPort: 3306, ContainerPort: 33060, ContainerPort: 33062, ContainerPort: 4444, ContainerPort: 4567, ContainerPort: 4568, HostIP: "", HostPort: 0, Items: nil, Items: nil, LocalObjectReference: {Name: "auto-some-name-pxc"}, LocalObjectReference: {Name: "some-name-pxc"}, Name: "ist", Name: "mysql", Name: "mysql-admin", Name: "mysqlx", Name: "sst", Name: "write-set", Optional: &true, Optional: &false, Optional: &true, SecretName: "internal-some-name", SecretName: "some-name-mysql-init", SecretName: "some-name-ssl", SecretName: "some-name-ssl-internal", SecretName: "some-name-vault", ... // 16 identical fields ... // 22 identical fields ... // 3 identical fields ... // 9 identical fields AWSElasticBlockStore: nil, AzureFile: nil, ConfigMap: &v1.ConfigMapVolumeSource{ FC: nil, FailureThreshold: 3, GitRepo: nil, ISCSI: nil, InitialDelaySeconds: 300, NFS: nil, ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}}, Quobyte: nil, Secret: &v1.SecretVolumeSource{ SuccessThreshold: 1, TerminationGracePeriodSeconds: nil, TimeoutSeconds: 5, VsphereVolume: nil, { }, ... // 16 identical fields ... // 3 identical fields ... // 4 identical fields Args: {"mysqld"}, Env: {{Name: "PXC_SERVICE", Value: "some-name-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-some-name"}, Key: "xtrabackup"}}}, ...}, EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "some-name-env-vars-pxc"}, Optional: &true}}}, ImagePullPolicy: "IfNotPresent", Lifecycle: nil, LivenessProbe: &v1.Probe{ Name: "auto-config", Name: "config", Name: "mysql-init-file", Name: "mysql-users-secret-file", Name: "ssl", Name: "ssl-internal", Name: "vault-keyring-secret", Ports: []v1.ContainerPort{ ReadinessProbe: &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...}, SecurityContext: nil, StartupProbe: nil, VolumeDevices: nil, VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...}, VolumeSource: v1.VolumeSource{ WorkingDir: "", }, ... // 3 identical fields ... // 5 identical fields AccessModes: nil, Capacity: nil, DataSource: nil, DataSourceRef: nil, StorageClassName: nil, VolumeAttributesClassName: nil, VolumeName: "", { {Name: "bin", VolumeSource: {EmptyDir: &{}}}, {Name: "tmp", VolumeSource: {EmptyDir: &{}}}, }, ... // 6 identical fields ... // 7 identical fields ActiveDeadlineSeconds: nil, Affinity: nil, AutomountServiceAccountToken: nil, Containers: []v1.Container{ EphemeralContainers: nil, HostAliases: nil, InitContainers: []v1.Container{ NodeName: "", NodeSelector: nil, OS: nil, ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, Overhead: nil, PreemptionPolicy: nil, ServiceAccountName: "default", SetHostnameAsFQDN: nil, Spec: v1.PersistentVolumeClaimSpec{ Status: v1.PersistentVolumeClaimStatus{ Subdomain: "", TerminationGracePeriodSeconds: &600, Tolerations: nil, TypeMeta: {}, Volumes: []v1.Volume{ }, ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "cfd6a52398268173b51d7cdb1331c09a", "percona.com/ssl-internal-hash": "751e59a1c1d94ac67f9d6bb6e756acdc"}}, Spec: v1.PodSpec{ { }, ... // 2 identical fields Conditions: nil, DeletionGracePeriodSeconds: nil, DeletionTimestamp: nil, Finalizers: nil, Labels: nil, MinReadySeconds: 0, Namespace: "sst-retry-limit-22208", Ordinals: nil, OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "some-name", UID: "6d4bc8d5-3c0b-4bfa-afaa-a00a2d08550a", ...}}, Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "some-name", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, SelfLink: "", ServiceName: "some-name-pxc", Template: v1.PodTemplateSpec{ UpdateStrategy: {Type: "RollingUpdate", RollingUpdate: &{Partition: &0}}, VolumeClaimTemplates: []v1.PersistentVolumeClaim{ }, ObjectMeta: v1.ObjectMeta{ Spec: v1.StatefulSetSpec{ Status: v1.StatefulSetStatus{ TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"}, }, &v1.StatefulSet{ } + DefaultMode: nil, + DefaultMode: nil, + Protocol: "", + PeriodSeconds: 0, + TerminationMessagePath: "", + TerminationMessagePolicy: "", + Phase: "", + VolumeMode: nil, + DNSPolicy: "", + DeprecatedServiceAccount: "", + RestartPolicy: "", + SchedulerName: "", + TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, + "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6NCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJl"..., + Annotations: map[string]string{ + AvailableReplicas: 0, + CollisionCount: nil, + CreationTimestamp: v1.Time{}, + CurrentReplicas: 0, + CurrentRevision: "", + Generation: 0, + ManagedFields: nil, + ObservedGeneration: 0, + PersistentVolumeClaimRetentionPolicy: nil, + PodManagementPolicy: "", + ReadyReplicas: 0, + Replicas: 0, + Replicas: &4, + ResourceVersion: "", + RevisionHistoryLimit: nil, + UID: "", + UpdateRevision: "", + UpdatedReplicas: 0, + }, - DefaultMode: &420, - DefaultMode: &420, - Protocol: "TCP", - PeriodSeconds: 10, - TerminationMessagePath: "/dev/termination-log", - TerminationMessagePolicy: "File", - Phase: "Pending", - VolumeMode: &"Filesystem", - APIVersion: "apps/v1", - APIVersion: "apps/v1", - DNSPolicy: "ClusterFirst", - DeprecatedServiceAccount: "default", - FieldsType: "FieldsV1", - FieldsType: "FieldsV1", - FieldsV1: s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`..., - FieldsV1: s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`..., - Manager: "kube-controller-manager", - Manager: "percona-xtradb-cluster-operator", - Operation: "Update", - Operation: "Update", - RestartPolicy: "Always", - SchedulerName: "default-scheduler", - Subresource: "status", - Time: s"2026-08-06 04:54:54 +0000 UTC", - Time: s"2026-08-06 04:50:59 +0000 UTC", - TopologySpreadConstraints: nil, - "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzb21lLW5hbWUiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJl"..., - { - }, - Annotations: map[string]string{ - AvailableReplicas: 3, - CollisionCount: &0, - CreationTimestamp: v1.Time{Time: s"2026-08-06 04:50:59 +0000 UTC"}, - CurrentReplicas: 3, - CurrentRevision: "some-name-pxc-64fc699588", - Generation: 1, - ManagedFields: []v1.ManagedFieldsEntry{ - ObservedGeneration: 1, - PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}", - PodManagementPolicy: "OrderedReady", - ReadyReplicas: 3, - Replicas: 3, - Replicas: &3, - ResourceVersion: "1785992094238015017", - RevisionHistoryLimit: &10, - UID: "316cd8c6-1557-47c2-b781-58e54979c972", - UpdateRevision: "some-name-pxc-64fc699588", - UpdatedReplicas: 3, - }, 2026-08-06T04:49:40.112Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-08-06T04:49:40.112Z INFO setup Manager starting up {"gitCommit": "c81e47f0baf68eddc8c1664c722d106d38b61d55", "gitBranch": "PR-2525-c81e47f0", "buildTime": "2026-08-06T01:12:58Z", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"} 2026-08-06T04:49:40.112Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.33.12-gke.1270000"} 2026-08-06T04:49:40.115Z INFO setup Registering Components. 2026-08-06T04:49:41.451Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-08-06T04:49:41.452Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-08-06T04:49:41.452Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-08-06T04:49:41.452Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-08-06T04:49:41.452Z INFO controller-runtime.metrics Starting metrics server 2026-08-06T04:49:41.452Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-08-06T04:49:41.452Z INFO controller-runtime.webhook Starting webhook server 2026-08-06T04:49:41.452Z INFO setup Starting the Cmd. 2026-08-06T04:49:41.452Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-08-06T04:49:41.553Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-08-06T04:49:41.581Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-08-06T04:49:41.582Z DEBUG events percona-xtradb-cluster-operator-6699755d74-b52dk_570b4fbe-be76-450a-9e64-d2e0496d38ce became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"56438134-aa6b-457f-936d-c6d4e2a36be6","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1785991781576191009"}, "reason": "LeaderElection"} 2026-08-06T04:49:41.582Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-08-06T04:49:41.582Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-08-06T04:49:41.582Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-08-06T04:49:41.582Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-08-06T04:49:41.683Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-08-06T04:49:41.683Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-08-06T04:49:41.783Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-08-06T04:49:41.783Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-08-06T04:49:41.783Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-08-06T04:49:41.783Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-08-06T04:50:58.757Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "3e608f51-25b1-46c8-b2ee-c87bd3a54588", "version": "1.21.0"} 2026-08-06T04:50:59.189Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "3e608f51-25b1-46c8-b2ee-c87bd3a54588", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-08-06T04:50:59.246Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "3e608f51-25b1-46c8-b2ee-c87bd3a54588", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-08-06T04:50:59.324Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "3e608f51-25b1-46c8-b2ee-c87bd3a54588", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-08-06T04:50:59.363Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "3e608f51-25b1-46c8-b2ee-c87bd3a54588", "object": "some-name-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-08-06T04:51:00.076Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "6a10529d-ef4b-43f1-a240-1f7b7c65a4a3", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-08-06T04:52:21.213Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b3426524-d403-4b01-be51-1100db129f04", "user": "operator"} 2026-08-06T04:52:21.251Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b3426524-d403-4b01-be51-1100db129f04", "user": "monitor"} 2026-08-06T04:52:21.308Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b3426524-d403-4b01-be51-1100db129f04"} 2026-08-06T04:52:21.357Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b3426524-d403-4b01-be51-1100db129f04"} 2026-08-06T04:52:21.412Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b3426524-d403-4b01-be51-1100db129f04", "user": "xtrabackup"} 2026-08-06T04:52:21.465Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b3426524-d403-4b01-be51-1100db129f04"} 2026-08-06T04:52:21.511Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b3426524-d403-4b01-be51-1100db129f04", "user": "replication"} 2026-08-06T04:52:21.514Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b3426524-d403-4b01-be51-1100db129f04", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:52:26.624Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "6430956c-feec-4cc1-a82e-2a78533ee788", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:52:31.735Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "94424141-9954-4ef8-b601-0b7988356019", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:52:36.847Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "4344dff0-d485-4291-bc28-aa468010fc1d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:52:41.972Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "5b920a3f-4160-4f11-9213-247e39c5a635", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:52:47.130Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "9c4b0fe4-3c5b-48af-a426-92db1180e6f4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:52:52.284Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "eb6fbd0b-9122-4172-b505-41a680d4d826", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:52:57.390Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "a3898df3-8a44-4c9b-9e2e-a0e8c5e73743", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:53:02.503Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "af7f84f2-d0e0-41b3-a825-573ce3f6b5a8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:53:07.620Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "f2d06b49-0664-4c47-85a5-2527544f9588", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:53:12.730Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "de9a2973-3a2c-4d78-8211-9e47384a897e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:53:17.833Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "1b3f9cfb-93b4-47e0-882b-30d98c52b617", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:53:22.940Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "1d7285bf-2781-497b-8f4e-7f55f23915a5", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:53:28.135Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "f540fe4d-bec9-40eb-8ab9-a3da55e79458", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:53:33.298Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "3aa67ec1-a7d0-42ab-b9f0-16ae418a8d6b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:53:38.414Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "c64912de-f8fc-4c56-bcd4-c70bc1578921", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:53:38.594Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "8a4a7321-1c42-4f08-a0e7-240206b0edb3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:53:43.734Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "39fc5e2b-3391-4c05-ab4c-fbc228dc8452", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:53:48.879Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "7fda903a-8d07-44a1-8d3d-7636bbd410d7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:53:54.045Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "a20d3154-35c6-4ab0-98a3-3f4afd08f488", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:53:59.213Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "19d894d4-c055-4de8-af56-96cb616b3d1a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:54:04.381Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "57c4ec67-b93c-4278-b084-16ea4ea8fcaa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:54:09.553Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "1eba5f0e-d8bb-4373-b785-21f399e78b94", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:54:14.694Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "16b3e5ea-01b7-4622-b854-082430dd0821", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:54:19.858Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "6a06380b-c5d3-4d18-8560-0eed1ac2c4a7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:54:25.419Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "c590cfb8-b821-4add-9ad9-c39f1d59a451", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:54:30.573Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "9bb2b1c2-fb40-4a1a-9cb3-3caf9019611c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:54:35.708Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "6cc3bae4-aa09-4bc6-b631-448d6d6f4a3b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:54:40.862Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "6976484c-94fd-468f-b437-54c3c2fe733f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:54:46.016Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "0196b663-9210-4aea-98d4-b3f3ab6c9642", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:54:51.167Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "83653880-9264-4545-9f63-6a7c17dce0e4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:54:56.364Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "e9b5f95e-8187-4140-a1bf-359b1454529e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:54:57.131Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "01e4cf6b-a497-4ee6-b035-57ee17e77462", "user": "root"} 2026-08-06T04:54:57.197Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "01e4cf6b-a497-4ee6-b035-57ee17e77462", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:54:57.204Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "01e4cf6b-a497-4ee6-b035-57ee17e77462", "new version": "8.0.43-34.1"} 2026-08-06T04:54:57.958Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "92f36291-4e27-4459-ba96-a09c1be30b4f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:55:03.647Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "50502064-cc8f-4a7e-b27f-42cdf274ce30", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:55:08.888Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "77f0cbcc-3f7e-4598-8846-0fb9a08dace7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:55:14.148Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "0704474f-7749-43ee-9dc5-be0331307e07", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:55:19.398Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "2d9246ab-2ce3-4f1e-ba37-cde363df2f97", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:55:24.614Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "112cef67-6e2b-4218-9837-8fc6b4fb6fb9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:55:29.835Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "bba72c0e-5b80-4fcb-a095-ad758358ecaa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:55:35.408Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "68dae5b6-bbe6-4ac9-82b5-6e6c6e7ffbb4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:55:40.675Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "37c97915-8f2f-45af-a462-029640a56374", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:55:45.891Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "aaba708a-f465-495f-856e-844f7227290a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:55:51.148Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "7dec9760-5203-4caf-9b16-f8540445df8c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:55:56.418Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "d72a4b08-2916-4dc2-9ce1-489eddf74f66", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:56:01.673Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "a10f3ac7-4c82-4b2c-84e5-8ce87e60cae4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:56:06.941Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "f5b10423-6010-4da9-8598-599273f7ad09", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:56:12.161Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "f68ce42e-c9ee-424f-aa3c-f0ca34712f29", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:56:17.382Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "1c0cd8bc-c1ff-4797-82bd-b524dbe8ea47", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:56:22.665Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "9a7fbda7-f636-4309-9b50-0daba2aa0cdc", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:56:27.889Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "35bdab27-8057-4d4a-9b00-ab251ea0330c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:56:33.351Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "3616e05e-51c5-4b66-8e8f-d2e714127bac", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:56:36.123Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "e091b94c-333d-4e50-b482-18c0c6d20219", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-08-06T04:56:36.180Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "e091b94c-333d-4e50-b482-18c0c6d20219", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-08-06T04:56:36.249Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "e091b94c-333d-4e50-b482-18c0c6d20219", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:56:37.028Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "389455d6-1416-4a25-956b-c175f1bfb4db", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:56:42.660Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "17f710ad-eb94-4f5d-8476-dd76cf423686", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:56:47.816Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "399cfce9-6b23-43e3-ab5f-f5b673c51678", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:56:53.005Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "04452196-503d-404a-b5c2-f3859c7bd8ec", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:56:58.248Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "d0c85f95-47dc-4e75-9da0-aab54afd55f7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:57:03.487Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "c4aefafd-da78-413a-8a44-d2d15028da51", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:57:08.700Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b0c7cd00-d9c6-4883-bfc1-6c8ae92bfcf9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:57:13.862Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "711478a6-6ec5-4a07-91f9-5fd940b4cd2c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:57:19.023Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "75573dc8-01b1-47d0-8f2b-2ef1b7fcc117", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:57:19.156Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "da50726a-aeb0-4266-9e44-aae6de8beedb", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:57:24.298Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "f0937b13-d212-4a44-b34f-aa9a2c75d7f2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:57:29.424Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "15df7fd3-6089-47f0-8025-9a821c6f0d8e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:57:34.533Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "d457b203-a92a-4b40-bd84-b24d6da06ad1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:57:39.649Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "9085cb0d-0944-4bda-9a66-843a1a5e97aa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:57:44.747Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "98faff0b-aebb-42f5-b06c-b96b0c1cd074", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:57:49.863Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b71e5389-8cab-4a48-b615-86be1a0ef269", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:57:54.988Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "555b99bf-8bdd-431d-9dc6-2747f2cfd3f1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:58:00.093Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "4bdee5a6-7cdd-4102-85c5-ab5c14a3a853", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:58:05.439Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "f0b5b83f-029b-4a27-846f-3a5ec551c58c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:58:05.624Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b834d8f3-53b2-4505-a92a-734931c4d0cd", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:58:11.079Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "eaadf723-80b2-47d7-93ea-a51a39abdc32", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:58:16.311Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "547661b7-a959-49dd-9943-4d7f6cf89b2a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:58:21.458Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "96477249-38ae-4c99-b347-d4f5f1b632ae", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:58:26.685Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "a5f3f8b6-33e4-42cd-a847-a9ddbd70211b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:58:31.854Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "4ec82ed7-7cad-414d-b442-e773d7c94de9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:58:37.034Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "246ac096-223b-4cb8-a06a-d19594400ac4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:58:42.207Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "a6bf0756-1925-4224-a74e-71f56ca02fb2", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:58:47.379Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "71c484b6-a693-4763-bb38-e8be3ac0c206", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:58:52.542Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "217eb4da-0ccf-42ab-a11c-1601e26bf9a7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:58:57.694Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "0d142644-bf0b-4f2e-99ca-66c1e2c26b42", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:59:02.849Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "3c96641e-75c9-4f9f-a252-ec1d21eec092", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:59:08.025Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "40606e9f-e9f6-4418-b0cb-9963a7dac7bc", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:59:13.317Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "dcd71f89-9880-40c2-8182-fab8f70f2ff0", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:59:18.513Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "f9a94212-95be-4c8e-8270-33eed267ad99", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:59:23.682Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "ef6622ad-1199-4ef8-bb18-6f60c20e4c35", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:59:28.854Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "60a0f503-dd24-4445-a737-dd72c9054665", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:59:34.031Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "20096545-9734-4a5c-bde0-5a31d3b61554", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:59:39.175Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "e3bef533-f998-4396-ad86-e69556857448", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:59:39.323Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "f4fdb84c-92c4-4dbb-b666-50834516eb03", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:59:44.452Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "2e71abda-0a3b-40f5-af11-db377f42fbed", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:59:49.560Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "c9548903-65e3-4bff-baad-c5ee632f6f54", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:59:54.669Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "d3314c53-6129-4370-9940-bf0b1691d364", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T04:59:59.788Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "4e2cbb4a-dac1-451a-92d7-9b2b886e153e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:00:05.139Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "a11fabc5-4f04-4675-b09d-9a855f88249d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:00:10.650Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "2fc1d713-b37a-431b-9fcf-2694004eb38d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:00:15.770Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "dac817b8-735a-414f-a88d-7b0756a774f1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:00:20.886Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "8f25d0bc-3e83-4eb9-91a2-fd54efef1044", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:00:25.984Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "eee7e56b-ea16-4f8b-8319-891020c65a1a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:00:31.113Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "358934da-774a-43b5-8490-9e2786ee663d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:00:31.287Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "9f0236e3-599a-4afd-a43b-fb9884d69b91", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:00:36.975Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "f02fda44-efa5-4714-a49d-7babb8161cbf", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:00:42.123Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "cf5d6c6c-306b-44ac-9d05-461d526f1fe6", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:00:47.303Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "bbf72650-6a74-45f0-be83-16c72d90e8d9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:00:52.482Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "24d3c0c6-873d-4e17-8b8b-67badc7edec7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:00:57.651Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "9445a20d-6ddf-430e-a9d0-bc05cb11da26", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:01:02.845Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "9f6bb2a2-6546-4544-8ac3-6a455abe93d8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:01:08.008Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "0472f421-0151-4a60-b92b-d2f996f64efa", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:01:13.258Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "5d48f15c-6ba0-4701-8c70-ed1085b89279", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:01:18.609Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "bb94c150-3f6d-4ca1-8f30-97e69fffc64d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:01:23.767Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "79ea9473-173a-4bc1-9a1d-4731f58630f8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:01:28.918Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "e9acfcde-846e-4b03-8287-83aa1ed4ed4f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:01:34.078Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "41c4fbc6-f328-4646-9538-ecfa709eca05", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:01:39.238Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "e21ed550-b2e5-4fad-a782-dfab579bfdc4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:01:44.397Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "06623df1-beb0-48e2-bc27-449eccdbe7c6", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:01:44.591Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "49f82eba-4563-42c7-a342-2cacfa3784d8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:01:50.427Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "25230501-a108-4962-802d-c6a64bdd4040", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:01:55.655Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "530fc145-8fdf-4f49-8fc9-17cc5fe636ff", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:02:00.824Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "320f9684-66ae-42fa-8e12-300a989a1207", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:02:01.139Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "57cdcdba-76b2-4e80-8f4a-ab677902167f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:02:06.710Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "f9957628-dc71-4a07-978f-1d9f2f2a4418", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:02:11.892Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "d1967435-5cb9-4508-b067-a2ec8f2d099e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:02:17.062Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "575bd96e-a685-4d4a-b6fc-119104037eab", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:02:22.238Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "074f8f23-1b8f-40e2-ad0b-88bef6926be7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:02:27.421Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b157013d-b8f0-4c5a-bc61-ac77d25d95f4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:02:32.578Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b1559654-8549-4f20-9f91-39350212155a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:02:37.728Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "42435973-9f74-439f-b86c-b152382c24a9", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:02:42.900Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "01512942-80c8-4c73-ad4a-e0318c5bfe77", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:02:48.077Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "bbe51e2d-ff81-4be2-985d-3c8ba5889dca", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:02:53.315Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "138930f8-23c3-479f-889e-318ecee15c6c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:02:58.532Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "7c1ab57d-b44b-48c1-94c8-21b96a6c884d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:03:03.816Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "385be2f6-7e36-4331-8c4e-7a84100fa622", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:03:08.997Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "78fd6ccb-e6e0-4369-aa1d-1e15c1c08494", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:03:14.158Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "5b156248-f10f-49bf-9ee1-09a41a042aa4", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:03:19.345Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "3c8a5497-24ca-4d18-a76a-3e413242df40", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:03:24.531Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "d177fd65-11c0-42cb-b6c9-fc6a61a9edbe", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:03:29.686Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "d5b003b7-0be0-4d7c-8734-deb048f4083d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:03:34.844Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b7ec73ec-ede4-4940-83e6-2f28f15e51ec", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:03:40.493Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "a58fabca-e839-406f-a53e-cecabe457d9e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:03:45.670Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "f85b4d5e-fade-4b33-b9b9-4bde25e5342b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:03:50.897Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "9c6fcc99-91f7-4356-bbf6-4ba581e4faf6", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:03:56.078Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "75f22b41-6d0f-427e-8474-57089d29efa1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:04:01.229Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "01018b2e-2874-400e-b94d-174de52e718f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:04:06.391Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b6889ce6-1469-4a24-bdc5-addad41e749d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:04:11.595Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "224486a0-6502-41ee-85e9-e0cb9a22ae2b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:04:17.030Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "d7a124bc-4044-45dc-b400-eb9915fde25a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:04:22.227Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "5f5048bc-837c-47c2-a989-66537eb59b9e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:04:27.426Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "16f4ffee-041f-4344-bc18-d943f0216972", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:04:32.600Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "47df5c9b-ef31-4748-8d1e-7d370d67689d", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:04:37.777Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "ef58aead-b0a7-4693-a36f-2fe899134d37", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:04:42.944Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "07f017ae-0f16-4c32-8381-bf9a8c22886f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:04:48.131Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "68e68fa5-d167-4f00-a39a-a04e3d8c8ba1", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:04:53.316Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "c8d2af86-0b43-4141-ab0b-f372d70cb3b7", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:04:58.502Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "3c8ee144-8b3b-482d-a3d4-89235b987eed", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:05:03.661Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "76d6d2ad-7dd8-4e30-8f72-7d41095b49c3", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:05:08.832Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "3520e367-3f26-4c3f-b0c3-588372295c74", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:05:13.993Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "c47828b3-c922-402e-befd-7427292fa6c8", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:05:19.143Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "b407ebf2-7143-406b-b54e-81c70c5d259c", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:05:24.293Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "4ceadf71-8398-42c1-a091-cdb57192a2cb", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:05:29.452Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "992c8626-480f-47a3-a5dd-7f23cff59375", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:05:34.641Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "ef09dbd9-1b68-4352-9cfe-52b27528636a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:05:39.843Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "df671c26-a424-4d63-a9a3-246ca19821cc", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:05:45.024Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "28f95805-190b-41af-bcfa-0d5faae8409f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:05:50.191Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "f36dac85-52a5-4acb-9a38-75cfa12ce14b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:05:55.534Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "22987135-c0ec-4e61-94d9-5a77afbd119e", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:06:00.791Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "caf4d37a-3e24-47b1-bc8f-a4a98e5ab219", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:06:01.587Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "75ac5455-2954-4e5e-bb56-6dbe2e09eb2b", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:06:07.524Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "3caa3610-524c-43cd-aff8-2af8082fc14a", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} 2026-08-06T05:06:12.787Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"sst-retry-limit-22208"}, "namespace": "sst-retry-limit-22208", "name": "some-name", "reconcileID": "ba9ff684-2d34-4d5a-8232-f097843d3d0f", "err": "get primary pxc pod: failed to get proxy connection: can't detect enabled proxy, please enable HAProxy or ProxySQL"} + local backup_names ++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + backup_names= + [[ -n '' ]] + local pxc_names ++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + pxc_names=sst-retry-limit-22208/some-name + [[ -n sst-retry-limit-22208/some-name ]] + read -r pns name + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + kubectl_bin patch pxc -n sst-retry-limit-22208 some-name --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' perconaxtradbcluster.pxc.percona.com/some-name patched + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --ignore-not-found perconaxtradbcluster.pxc.percona.com "some-name" deleted from sst-retry-limit-22208 namespace ++ date +%s + local deadline=1785993087 ++ kubectl_bin get pxc --all-namespaces -o name --ignore-not-found + [[ -z '' ]] + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.BdJzWc6RKY ++ mktemp + local LAST_ERR=/tmp/tmp.mmWtrN1abo + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BdJzWc6RKY validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.mmWtrN1abo + rm /tmp/tmp.BdJzWc6RKY /tmp/tmp.mmWtrN1abo + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' ++ date +%s + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator + local deadline=1785992936 + kubectl_bin delete --grace-period=0 --force=true namespace sst-retry-limit-22208 ++ mktemp ++ kubectl_bin get namespace sst-retry-limit-22208 --ignore-not-found -o name + local LAST_OUT=/tmp/tmp.9qjwkrAPiS ++ mktemp + local LAST_ERR=/tmp/tmp.ruvxFgczLf + local exit_status=0 ++ seq 0 2 ++ mktemp + local LAST_OUT=/tmp/tmp.AjaxTZBAV8 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp + local LAST_ERR=/tmp/tmp.HGBPCtqbiq + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace sst-retry-limit-22208 + [[ -z namespace/sst-retry-limit-22208 ]] ++ date +%s + [[ 1785992821 -ge 1785992936 ]] + sleep 5 ++ kubectl_bin get namespace sst-retry-limit-22208 --ignore-not-found -o name + [[ -z namespace/sst-retry-limit-22208 ]] ++ date +%s + [[ 1785992828 -ge 1785992936 ]] + sleep 5 ++ kubectl_bin get namespace sst-retry-limit-22208 --ignore-not-found -o name + [[ -z namespace/sst-retry-limit-22208 ]] ++ date +%s + [[ 1785992836 -ge 1785992936 ]] + sleep 5 ++ kubectl_bin get namespace sst-retry-limit-22208 --ignore-not-found -o name + [[ -z namespace/sst-retry-limit-22208 ]] ++ date +%s + [[ 1785992843 -ge 1785992936 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AjaxTZBAV8 namespace "sst-retry-limit-22208" force deleted + cat /tmp/tmp.HGBPCtqbiq Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.AjaxTZBAV8 /tmp/tmp.HGBPCtqbiq + return 0 ++ kubectl_bin get namespace sst-retry-limit-22208 --ignore-not-found -o name + [[ -z '' ]] ++ date +%s + deadline=1785992970 ++ awk '$2 != ""' ++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' + [[ -z '' ]] + rm -rf /tmp/tmp.xcRCCWmxWa + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------